diff --git a/apps/vms/unifi/30-virtualmachine.yaml b/apps/vms/unifi/30-virtualmachine.yaml index ae0d159..6d99442 100644 --- a/apps/vms/unifi/30-virtualmachine.yaml +++ b/apps/vms/unifi/30-virtualmachine.yaml @@ -102,7 +102,7 @@ spec: userData: |- #cloud-config hostname: unifi - fqdn: unifi.home.keithjackson.dev + fqdn: unifi.${BASE_DOMAIN} package_update: true package_upgrade: false packages: [qemu-guest-agent, podman, uidmap, slirp4netns, ca-certificates, curl, jq, cloud-guest-utils] diff --git a/apps/vms/unifi/50-service-l4.yaml b/apps/vms/unifi/50-service-l4.yaml index a380ee6..cc68052 100644 --- a/apps/vms/unifi/50-service-l4.yaml +++ b/apps/vms/unifi/50-service-l4.yaml @@ -46,7 +46,7 @@ metadata: lbipam.cilium.io/ips: "${UNIFI_L4_VIP}" # external-dns (source: service) publishes A unifi.home.keithjackson.dev -> the pinned VIP. # This is the browser + device target; UI is https://unifi.home.keithjackson.dev:11443. - external-dns.alpha.kubernetes.io/hostname: "unifi.home.keithjackson.dev" + external-dns.alpha.kubernetes.io/hostname: "unifi.${BASE_DOMAIN}" spec: type: LoadBalancer externalTrafficPolicy: Local diff --git a/apps/wave-0/gitea.yaml b/apps/wave-0/gitea.yaml index 8eff421..dac5a59 100644 --- a/apps/wave-0/gitea.yaml +++ b/apps/wave-0/gitea.yaml @@ -90,9 +90,9 @@ spec: name: gitea-bootstrap key: secret-key - name: GITEA__server__DOMAIN - value: gitea.home.keithjackson.dev + value: gitea.${BASE_DOMAIN} - name: GITEA__server__ROOT_URL - value: https://gitea.home.keithjackson.dev/ + value: https://gitea.${BASE_DOMAIN}/ - name: GITEA__actions__ENABLED value: 'true' - name: GITEA__actions__DEFAULT_ACTIONS_URL @@ -127,7 +127,7 @@ spec: command: - /bin/sh - -c - - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"$GITEA_ADMIN_USERNAME@home.local\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\ + - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"$GITEA_ADMIN_USERNAME@home.local\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\ \ break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n# A postStart hook MUST exit 0 \u2014 a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff). This hook already ends 0 on both paths, but\n# it is made EXPLICIT so the guarantee does not depend on whatever statement happens to\n# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,\n# the moment its OIDC guard was fixed and the block actually ran for the first time.\nexit 0\n" volumes: - name: data @@ -207,7 +207,7 @@ spec: namespace: gateway sectionName: https hostnames: - - gitea.home.keithjackson.dev + - gitea.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-0/jackett.yaml b/apps/wave-0/jackett.yaml index f80c721..b445113 100644 --- a/apps/wave-0/jackett.yaml +++ b/apps/wave-0/jackett.yaml @@ -91,16 +91,16 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:9117 - --email-domain=* - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users - --skip-provider-button=true - - --redirect-url=https://jackett.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true @@ -273,7 +273,7 @@ spec: namespace: gateway sectionName: https hostnames: - - jackett.home.keithjackson.dev + - jackett.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-0/pihole.yaml b/apps/wave-0/pihole.yaml index e60eb93..680e52c 100644 --- a/apps/wave-0/pihole.yaml +++ b/apps/wave-0/pihole.yaml @@ -91,16 +91,16 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:80 - --email-domain=* - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=platform-admins - --skip-provider-button=true - - --redirect-url=https://pihole.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://pihole.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true @@ -229,7 +229,7 @@ spec: namespace: gateway sectionName: https hostnames: - - pihole.home.keithjackson.dev + - pihole.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-0/plex.yaml b/apps/wave-0/plex.yaml index 8762e84..6575b7b 100644 --- a/apps/wave-0/plex.yaml +++ b/apps/wave-0/plex.yaml @@ -183,7 +183,7 @@ spec: namespace: gateway sectionName: https hostnames: - - plex.home.keithjackson.dev + - plex.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-0/qbittorrent.yaml b/apps/wave-0/qbittorrent.yaml index 573794a..a4e92ba 100644 --- a/apps/wave-0/qbittorrent.yaml +++ b/apps/wave-0/qbittorrent.yaml @@ -91,16 +91,16 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:8080 - --email-domain=* - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users - --skip-provider-button=true - - --redirect-url=https://qbittorrent.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true @@ -273,7 +273,7 @@ spec: namespace: gateway sectionName: https hostnames: - - qbittorrent.home.keithjackson.dev + - qbittorrent.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-1/gitea-actions.yaml b/apps/wave-1/gitea-actions.yaml index 198edb5..5b9a32e 100644 --- a/apps/wave-1/gitea-actions.yaml +++ b/apps/wave-1/gitea-actions.yaml @@ -165,7 +165,7 @@ spec: exec /sbin/tini -- /opt/act/run.sh env: - name: GITEA_INSTANCE_URL - value: https://gitea.home.keithjackson.dev + value: https://gitea.${BASE_DOMAIN} - name: GITEA_RUNNER_REGISTRATION_TOKEN valueFrom: secretKeyRef: diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index 3b09759..eb5a024 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -104,13 +104,13 @@ spec: name: nextcloud-bootstrap key: admin-password - name: NEXTCLOUD_TRUSTED_DOMAINS - value: nextcloud.home.keithjackson.dev + value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEHOST - value: nextcloud.home.keithjackson.dev + value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEPROTOCOL value: https - name: OVERWRITECLIURL - value: https://nextcloud.home.keithjackson.dev + value: https://nextcloud.${BASE_DOMAIN} - name: NEXTCLOUD_OIDC_CLIENT_SECRET valueFrom: secretKeyRef: @@ -125,9 +125,9 @@ spec: command: - /bin/sh - -c - - "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"\ - [nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# ` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"\ - $ADMIN_SRC_GROUP\"); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n" + - "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[nextcloud-oidc]\ + \ client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# ` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"$ADMIN_SRC_GROUP\"\ + ); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n" volumes: - name: data persistentVolumeClaim: @@ -228,7 +228,7 @@ spec: namespace: gateway sectionName: https hostnames: - - nextcloud.home.keithjackson.dev + - nextcloud.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index 38a6186..021bdd4 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -91,16 +91,16 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:7878 - --email-domain=* - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users - --skip-provider-button=true - - --redirect-url=https://radarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true @@ -295,7 +295,7 @@ spec: namespace: gateway sectionName: https hostnames: - - radarr.home.keithjackson.dev + - radarr.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-1/semaphore.yaml b/apps/wave-1/semaphore.yaml index 40b6388..d29a2e6 100644 --- a/apps/wave-1/semaphore.yaml +++ b/apps/wave-1/semaphore.yaml @@ -110,7 +110,7 @@ spec: - name: SEMAPHORE_ADMIN_NAME value: Semaphore Admin - name: SEMAPHORE_ADMIN_EMAIL - value: admin@home.keithjackson.dev + value: admin@${BASE_DOMAIN} - name: SEMAPHORE_ACCESS_KEY_ENCRYPTION valueFrom: secretKeyRef: @@ -231,7 +231,7 @@ spec: namespace: gateway sectionName: https hostnames: - - semaphore.home.keithjackson.dev + - semaphore.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 39d8dd7..9af2717 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -91,16 +91,16 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:8989 - --email-domain=* - --scope=openid email profile - --oidc-groups-claim=groups - --allowed-group=media-users - --skip-provider-button=true - - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true @@ -295,7 +295,7 @@ spec: namespace: gateway sectionName: https hostnames: - - sonarr.home.keithjackson.dev + - sonarr.${BASE_DOMAIN} rules: - matches: - path: diff --git a/apps/wave-2/seerr.yaml b/apps/wave-2/seerr.yaml index 3fdf212..9d6e171 100644 --- a/apps/wave-2/seerr.yaml +++ b/apps/wave-2/seerr.yaml @@ -223,7 +223,7 @@ spec: namespace: gateway sectionName: https hostnames: - - seerr.home.keithjackson.dev + - seerr.${BASE_DOMAIN} rules: - matches: - path: diff --git a/catalog/gitea.yaml b/catalog/gitea.yaml index d98e0c5..36d76ae 100644 --- a/catalog/gitea.yaml +++ b/catalog/gitea.yaml @@ -24,9 +24,9 @@ env: name: gitea-bootstrap key: secret-key - name: GITEA__server__DOMAIN # correct public clone/redirect URLs for the TLS-terminated host - value: gitea.home.keithjackson.dev + value: gitea.${BASE_DOMAIN} - name: GITEA__server__ROOT_URL - value: https://gitea.home.keithjackson.dev/ + value: https://gitea.${BASE_DOMAIN}/ - name: GITEA__actions__ENABLED # Phase 6 CI/CD: declared (not implicit upstream default) so the value: "true" # in-cluster act_runner (ns gitea-actions) registers against it - name: GITEA__actions__DEFAULT_ACTIONS_URL # resolve `uses: /@ref` from github.com (cluster @@ -104,7 +104,7 @@ lifecycle: # persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker # file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays # usable). The verify-then-retry loop is the real guarantee against "exit 0 but nothing stuck". - GITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration + GITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration gitea_oidc_fail() { echo "[gitea-oidc] FAILED to register 'keycloak' login source: $1" >&2 { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true @@ -168,7 +168,7 @@ lifecycle: # be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01, # the moment its OIDC guard was fixed and the block actually ran for the first time. exit 0 -exposure: public # edge-forwarded git host at gitea.home.keithjackson.dev (docs/09; R-DNS/R-NET) +exposure: public # edge-forwarded git host at gitea.${BASE_DOMAIN} (docs/09; R-DNS/R-NET) egress: internet # normal WAN egress (docs/03; R-NET-05) storage: - name: data @@ -178,7 +178,7 @@ storage: # the default /data/ would strand real state on the ephemeral rootfs (R-STATE-01) dependencies: [] # default-deny: reaches only DNS; git clients reach it north-south via the Gateway (R-NET-01/02) ingressRoutes: - - host: gitea.home.keithjackson.dev + - host: gitea.${BASE_DOMAIN} port: 3000 # Gitea HTTP web/API; TLS terminates at the shared Gateway (docs/08, docs/09) auth: mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02) diff --git a/catalog/jackett.yaml b/catalog/jackett.yaml index 3ae036e..d09752f 100644 --- a/catalog/jackett.yaml +++ b/catalog/jackett.yaml @@ -17,7 +17,7 @@ name: jackett kind: container source: image: lscr.io/linuxserver/jackett:latest -exposure: internal # LAN-only web UI at jackett.home.keithjackson.dev (docs/13 row 4; R-DNS/R-NET) +exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET) egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06) storage: - name: config @@ -25,7 +25,7 @@ storage: class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02) dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02) ingressRoutes: - - host: jackett.home.keithjackson.dev + - host: jackett.${BASE_DOMAIN} port: 9117 # Jackett WebUI; protected at the Gateway by forward-auth (docs/09, docs/15) auth: mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02) diff --git a/catalog/keycloak.yaml b/catalog/keycloak.yaml index eabc4ed..21f482d 100644 --- a/catalog/keycloak.yaml +++ b/catalog/keycloak.yaml @@ -18,7 +18,7 @@ dependencies: port: 5432 protocol: tcp ingressRoutes: - - host: keycloak.home.keithjackson.dev + - host: keycloak.${BASE_DOMAIN} port: 8080 # HTTP listener; TLS terminates at the Gateway (docs/08, docs/09) auth: mode: native # it IS the IdP; own admin login is local + MFA (docs/15; ADR-0014). Documented exception. diff --git a/catalog/nextcloud.yaml b/catalog/nextcloud.yaml index 081afc2..ab1e40d 100644 --- a/catalog/nextcloud.yaml +++ b/catalog/nextcloud.yaml @@ -50,13 +50,13 @@ env: # the vhost answer for the public FQDN; OVERWRITE* make Nextcloud emit https:// self-URLs even # though it speaks plain HTTP behind the proxy. --- - name: NEXTCLOUD_TRUSTED_DOMAINS - value: nextcloud.home.keithjackson.dev + value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEHOST - value: nextcloud.home.keithjackson.dev + value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEPROTOCOL value: https - name: OVERWRITECLIURL - value: https://nextcloud.home.keithjackson.dev + value: https://nextcloud.${BASE_DOMAIN} # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential # `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret # (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it @@ -88,7 +88,7 @@ lifecycle: # provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as # ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument. OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; } - NEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration + NEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration nextcloud_oidc_fail() { echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2 { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true @@ -228,7 +228,7 @@ dependencies: port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress. ingressRoutes: - - host: nextcloud.home.keithjackson.dev + - host: nextcloud.${BASE_DOMAIN} port: 80 # apache HTTP; wildcard TLS terminates at the shared Gateway (docs/08/09) auth: mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02) diff --git a/catalog/pihole.yaml b/catalog/pihole.yaml index 43fd39d..e64bae4 100644 --- a/catalog/pihole.yaml +++ b/catalog/pihole.yaml @@ -14,7 +14,7 @@ storage: class: longhorn # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03) dependencies: [] # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02) ingressRoutes: - - host: pihole.home.keithjackson.dev + - host: pihole.${BASE_DOMAIN} port: 80 # admin web UI, protected at the Gateway by forward-auth (docs/09) auth: mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02) diff --git a/catalog/plex.yaml b/catalog/plex.yaml index 87886f9..c2b9f0d 100644 --- a/catalog/plex.yaml +++ b/catalog/plex.yaml @@ -72,7 +72,7 @@ storage: # so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off. dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02) ingressRoutes: - - host: plex.home.keithjackson.dev + - host: plex.${BASE_DOMAIN} port: 32400 # Plex web UI + API served on 32400 (wildcard TLS at the Gateway; docs/08/09) auth: mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02) diff --git a/catalog/qbittorrent.yaml b/catalog/qbittorrent.yaml index 307be99..a9ceef6 100644 --- a/catalog/qbittorrent.yaml +++ b/catalog/qbittorrent.yaml @@ -32,7 +32,7 @@ name: qbittorrent kind: container source: image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement -exposure: internal # LAN-only web UI at qbittorrent.home.keithjackson.dev (docs/13 row 13; R-DNS/R-NET) +exposure: internal # LAN-only web UI at qbittorrent.${BASE_DOMAIN} (docs/13 row 13; R-DNS/R-NET) egress: vpn # ALL public egress via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06) storage: - name: config @@ -40,7 +40,7 @@ storage: class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02) dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02) ingressRoutes: - - host: qbittorrent.home.keithjackson.dev + - host: qbittorrent.${BASE_DOMAIN} port: 8080 # qBittorrent WebUI; protected at the Gateway by forward-auth (docs/09, docs/15) auth: mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02) diff --git a/catalog/radarr.yaml b/catalog/radarr.yaml index 8ce6c89..ae385f8 100644 --- a/catalog/radarr.yaml +++ b/catalog/radarr.yaml @@ -9,7 +9,7 @@ name: radarr kind: container source: image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement -exposure: internal # LAN-only web UI at radarr.home.keithjackson.dev (docs/13 row 3; R-DNS/R-NET) +exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) storage: @@ -29,7 +29,7 @@ dependencies: port: 9117 protocol: tcp ingressRoutes: - - host: radarr.home.keithjackson.dev + - host: radarr.${BASE_DOMAIN} port: 7878 # Radarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09) auth: mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02) diff --git a/catalog/render.py b/catalog/render.py index 992f531..16b4323 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -44,7 +44,15 @@ except ImportError: # pragma: no cover - explicit, actionable failure # --- Label / naming conventions (match platform/kyverno/policies + platform/*/) -------- LBL = "homelab.keithjackson.dev" -ZONE = "home.keithjackson.dev" +# The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal. +# Every rendered manifest lands under a Flux path whose Kustomization declares +# postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at +# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev. +# It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE — +# the escape contract below reserves `$${...}` for expansions meant for a container's shell. +# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames, +# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10). +ZONE = "${BASE_DOMAIN}" GATEWAY_NAME = "shared" GATEWAY_NS = "gateway" KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label diff --git a/catalog/seerr.yaml b/catalog/seerr.yaml index e5e2c31..5fa0fb2 100644 --- a/catalog/seerr.yaml +++ b/catalog/seerr.yaml @@ -57,7 +57,7 @@ dependencies: port: 32400 protocol: tcp ingressRoutes: - - host: seerr.home.keithjackson.dev + - host: seerr.${BASE_DOMAIN} port: 5055 # Overseerr web UI + API; wildcard TLS terminates at the shared Gateway (docs/08/09) auth: mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02) diff --git a/catalog/semaphore.yaml b/catalog/semaphore.yaml index fa4ac02..34ae9b4 100644 --- a/catalog/semaphore.yaml +++ b/catalog/semaphore.yaml @@ -58,7 +58,7 @@ env: - name: SEMAPHORE_ADMIN_NAME value: Semaphore Admin - name: SEMAPHORE_ADMIN_EMAIL - value: admin@home.keithjackson.dev + value: admin@${BASE_DOMAIN} # --- Persistent access-key encryption key (see header): FIXED base64 32-byte key from SOPS so the # SSH keys / credentials Semaphore stores in the DB stay decryptable across restarts. --- - name: SEMAPHORE_ACCESS_KEY_ENCRYPTION @@ -110,7 +110,7 @@ dependencies: port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress. ingressRoutes: - - host: semaphore.home.keithjackson.dev + - host: semaphore.${BASE_DOMAIN} port: 3000 # Semaphore HTTP web/API; wildcard TLS terminates at the shared Gateway (docs/08/09) auth: mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02) diff --git a/catalog/sonarr.yaml b/catalog/sonarr.yaml index df87757..25cc318 100644 --- a/catalog/sonarr.yaml +++ b/catalog/sonarr.yaml @@ -9,7 +9,7 @@ name: sonarr kind: container source: image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement -exposure: internal # LAN-only web UI at sonarr.home.keithjackson.dev (docs/13 row 2; R-DNS/R-NET) +exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET) egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway, # kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06) storage: @@ -29,7 +29,7 @@ dependencies: port: 9117 protocol: tcp ingressRoutes: - - host: sonarr.home.keithjackson.dev + - host: sonarr.${BASE_DOMAIN} port: 8989 # Sonarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09) auth: mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02) diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index ee03139..0592022 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -17,9 +17,63 @@ data: CLUSTER_ENV: "base" # dev | prod (overlays set this) # --- DNS / TLS --- - BASE_DOMAIN: "home.keithjackson.dev" # platform hostname suffix (same both envs; split-horizon) - DNS_ZONE: "keithjackson.dev" # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it); - # ExternalDNS domain-filter must be the zone, not the subdomain + # THE platform hostname suffix. Every public hostname the cluster serves is `.${BASE_DOMAIN}`, + # the wildcard cert is `*.${BASE_DOMAIN}`, coredns-custom answers the `${BASE_DOMAIN}` zone with + # ${GATEWAY_VIP}, and every OIDC issuer/redirect URI is built from it. + # + # THIS KNOB WAS DEAD UNTIL 2026-09-04 — declared here but referenced by NO manifest, because every + # hostname was hardcoded. That is what made the 2026-09-04 incident possible: prod and dev were + # authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns + # (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM. + # See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN + # subdomain, so the two clusters can no longer describe the same name. + # + # dev -> home.keithjackson.dev (UNCHANGED — see below) + # prod -> prod.home.keithjackson.dev + # + # DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on + # `dev.home.keithjackson.dev` with apex CNAMEs `.home` -> `.prod.home`, but moving dev + # relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and + # the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2 + # (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as + # GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates + # it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre- + # tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths). + BASE_DOMAIN: "home.keithjackson.dev" + + # ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind + # EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered, + # so it cannot be created, updated or deleted regardless of TXT ownership. + # + # DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev` + # apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter + # that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching + # record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged + # from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically. + # + # PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE + # (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned + # chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED + # — it can never delete another environment's records — and the fallback is to set prod's value + # back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard. + EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev" + + # ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT: + # neither cluster set it, so both inherited the chart default `default`, and ExternalDNS treats a + # record whose TXT registry entry carries ITS OWN owner id as its own to reconcile — and with + # `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on + # one zone therefore garbage-collect each other. Each environment MUST have a distinct value. + # + # DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare + # under owner `default`; changing dev's value would make dev's own external-dns stop recognising + # them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it. + # + # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that + # forgets this knob must not silently inherit dev's identity — that is precisely the incident. With + # this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a + # visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS. + EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars" + ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006) GATEWAY_VIP: "192.168.1.53" # in-cluster split-horizon target (R-DNS-01/03): the L2-announced # ingress VIP of the `shared` Gateway. MUST equal the FIRST address of diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 33d180e..1708eab 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -148,6 +148,33 @@ patches: - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" + # --- DNS identity (R-DNS-01/02/03) ----------------------------------------------------- + # DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly + # like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running. + # Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its + # Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev + # (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to + # prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did + # not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration, + # not a variable change: infra/PROD-PROVISIONING.md §10. + - op: replace + path: /data/BASE_DOMAIN + value: "home.keithjackson.dev" + # ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself, + # and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped + # the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held. + - op: replace + path: /data/EXTERNALDNS_DOMAIN_FILTER + value: "keithjackson.dev" + # DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT + # registry under owner `default` (the external-dns chart default, which is what both clusters + # silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records + # BY this string: any other value and it stops recognising every record it manages, then + # re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the + # whole point. The base default is a non-colliding sentinel, so this line is load-bearing. + - op: replace + path: /data/EXTERNALDNS_TXT_OWNER_ID + value: "default" # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) ----------------------------- # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it # equals the base default. These values are what the LIVE dev cluster is running right now, so diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index aaaeb3d..c7d7efc 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -201,6 +201,63 @@ patches: - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" + # ========================================================================================== + # PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident. + # ========================================================================================== + # PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to + # home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME + # public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one + # of which already happened: + # 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit + # default dev had) concluded it owned dev's records and DELETED EVERY ONE of them. + # 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev` + # wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would + # have raced each other's `_acme-challenge` TXT records on that one zone. + # Moving prod to its own suffix dissolves both: the two clusters can no longer name the same + # record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here. + # + # dev -> home.keithjackson.dev (UNCHANGED — dev is live; phase 2 moves it) + # prod -> prod.home.keithjackson.dev (this knob) + # + # CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is + # gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so + # on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make + # `.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they + # would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10. + # + # [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml. + # The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is + # set Ansible-side from that variable and cannot read this one. If they disagree, kubectl SSO + # trusts a different Keycloak than the one prod actually runs. + - op: replace + path: /data/BASE_DOMAIN + value: "prod.home.keithjackson.dev" + # SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's + # OWN subtree, so prod cannot create, update or delete any record outside it no matter what the + # TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for + # ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped + # the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no + # matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's + # records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on + # EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own. + - op: replace + path: /data/EXTERNALDNS_DOMAIN_FILTER + # MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG, + # 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the + # record then skipped it with "no hosted zone matching record DNS Name", so prod would + # publish NOTHING. Isolation from dev does NOT come from this filter — it comes from + # (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and + # (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records. + # That pairing is what the 2026-09-04 incident needed and this filter cannot provide. + value: "keithjackson.dev" + # SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so + # both inherited the chart default `default`; prod then treated dev's TXT-registry entries as + # its own and, under policy: sync, deleted the records behind them. This value MUST differ from + # dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and + # cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix. + - op: replace + path: /data/EXTERNALDNS_TXT_OWNER_ID + value: "prod" - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT value: "3" diff --git a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml index 002f573..857479e 100644 --- a/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml +++ b/conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml @@ -68,7 +68,7 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:8080 - --email-domain=* - --scope=openid email profile @@ -76,8 +76,8 @@ spec: - --allowed-group=platform-admins - --skip-provider-button=true - --redirect-url=https://paas-demo.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml index 4fb453d..4aa6b62 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml @@ -91,7 +91,7 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:8989 - --email-domain=* - --scope=openid email profile @@ -99,8 +99,8 @@ spec: - --allowed-group=media-admins - --skip-provider-button=true - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true diff --git a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml index db2aeb9..0980eca 100644 --- a/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml +++ b/conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml @@ -91,7 +91,7 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab - --upstream=http://127.0.0.1:8989 - --email-domain=* - --scope=openid email profile @@ -99,8 +99,8 @@ spec: - --allowed-group=media-admins - --skip-provider-button=true - --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true diff --git a/infra/PROD-PROVISIONING.md b/infra/PROD-PROVISIONING.md index 99bbbd0..2e7a9e4 100644 --- a/infra/PROD-PROVISIONING.md +++ b/infra/PROD-PROVISIONING.md @@ -1172,6 +1172,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be | `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address | | `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address | | `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target | +| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. | - [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both* overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the @@ -1282,3 +1283,152 @@ single-server-etcd window, and avoids a later member-add dance. specifically. - [ ] Monitoring and alerts live on prod. - [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03). + Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2. +- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10): + distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and + prod's own Keycloak realm + per-service client secrets. + +## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix** + +### What happened + +Prod's `external-dns` **deleted every one of dev's public DNS records.** + +Both clusters ran `policy: sync` against the **same Cloudflare zone**, served the **same +hostnames**, and — because neither overlay set `txtOwnerId` — both inherited the external-dns +chart's default owner id **`default`**. ExternalDNS decides what it may delete by reading its own +owner id back out of the TXT registry. Prod read dev's TXT entries, saw its own owner id, found no +local resource claiming those names, and garbage-collected the records behind them. Dev recovered +once prod was stopped. Prod's `platform` tier and everything downstream were then **suspended**, and +remain suspended, so it cannot recur before this change is reviewed. + +`policy: sync` was not the bug and has not been changed. What was missing was **identity**. + +### The second bug, queued directly behind it + +`platform-config` would have requested a `*.home.keithjackson.dev` wildcard `Certificate` from +**letsencrypt-prod** — for a cert **dev already holds** — and both clusters would then have raced +each other's `_acme-challenge` TXT records on that one zone. Neither would reliably solve DNS-01. +This never fired only because the tier was suspended first. + +### Root cause, stated precisely + +Not external-dns. **Every hostname in the repo was a hardcoded literal.** `BASE_DOMAIN` existed in +`clusters/base/cluster-vars.yaml` but was referenced by **no reconciled manifest** — the repo-derived +status report flagged it as a dead knob. R-ENV-02 promises dev and prod differ only by `cluster-vars` +values; for DNS and TLS that promise was false, so `clusters/prod` was *authored* to serve dev's +public identity. Two clusters that describe the same DNS name will fight over it no matter how the +DNS controller is configured. + +### The architecture decision (operator) + +``` +dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev +prod -> *.prod.home.keithjackson.dev gitea.prod.home.keithjackson.dev +apex -> CNAME .home.keithjackson.dev -> .prod.home.keithjackson.dev +``` + +### PHASE 1 — **prod only. Dev did not move.** (this change) + +`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it. + +| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | +|---|---|---|---| +| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | +| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` | +| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` | + +**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod +alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken +while the operator is away. Dev's rendered output is **byte-identical** except for one added line, +`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified +by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for +both overlays and diffing. + +**Two independent guards now, either sufficient alone:** + +1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is + the direct fix for the incident. Dev **must** render exactly `default`: its live records are + already registered under that owner, and any other value would make dev's own external-dns stop + recognising them and re-create duplicates beside the orphans. +2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree. + Defence in depth, added because the first guard is a single string in a single file. + +The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a +future environment that forgets the knob must not silently inherit dev's identity — that is exactly +what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing. + +### What could NOT be tokenized, and why + +- **`OIDC_ISSUER_URL` / `OIDC_CALLBACK_URL` in `platform/headlamp/headlamp-oidc-credentials.sops.yaml`** + — SOPS-encrypted (`encrypted_regex: ^(data|stringData)$`), so the two hostnames are ciphertext. + Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and + **cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake + would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately. + **Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak. + Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client + secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret + is per-realm. +- **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s + **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution + can never reach it. Correctly tokenized on the **Ansible** side instead: new role default + `k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in + `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`. + `[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver + validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the + cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew). + +### A pre-existing prod problem this incidentally FIXED + +§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC +issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN, +making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on +its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by +**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer +and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's +external-dns has actually published the record; until then use the break-glass cert kubeconfig. + +### Verify on prod's first reconcile + +- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup + as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone + ("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it + will create **nothing** — fail-closed, never destructive. Fallback: set prod's + `EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is + sufficient on its own. +- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching + dev's `_acme-challenge` records. +- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry + entries still read owner `default` and that prod's read owner `prod`. +- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client + secret is realm-scoped; prod cannot reuse dev's). + +### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**) + +Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the +Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change. +Required, roughly in order: + +1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the + dev inventory group. +2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover. +3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client + (`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, + `qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs + updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a + **rolling k3s restart** on the dev servers. +4. **kubectl SSO re-issue** for every user (issuer changed). +5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in + `clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before* + the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself. + Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05). +6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned + records are for names dev no longer serves; retire them deliberately rather than letting a + renamed owner orphan them. +7. **Apex CNAMEs** `.home.keithjackson.dev` → `.prod.home.keithjackson.dev`. This is what + actually **moves user traffic to prod**, and it must not land before prod serves those services. +8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`, + `dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`, + `trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1 + — but they will fail against prod and must be parameterized before §9's "suite passes green + against the prod cluster" can be claimed. diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index a281631..056ae85 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -289,19 +289,40 @@ node_baseline_dns_servers: # LAN clients -> PiHole. Not a cluster concern. # nodes -> THIS FILE. Public resolvers, so the zone resolves PUBLICLY. # - # THE ONE THING THAT BREAKS: the k3s API SERVER is a node process and carries - # `oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab` - # (roles/k3s_server/tasks/main.yml). It reaches the issuer through the NODE resolver, so on - # prod it would get the PUBLIC record instead of the prod Gateway VIP — breaking kubectl - # OIDC SSO. Dev avoids this because its node resolver is PiHole, which answers the zone - # with the dev Gateway VIP. + # THE k3s API SERVER is a node process and carries `oidc-issuer-url=...` (built from + # k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through + # these PUBLIC upstreams — not through CoreDNS's split-horizon answer. # - # NOT blocking today (prod has no cluster and no Keycloak yet). Options, in - # PROD-PROVISIONING.md §4: a narrow Ansible-managed /etc/hosts entry pinning - # keycloak.home.keithjackson.dev -> 192.168.2.20 on prod nodes; or a local resolver on the - # prod LAN serving the zone; or run prod on the cert kubeconfig and defer OIDC. + # THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was + # authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev + # publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so + # kubectl OIDC SSO on prod could not work at all. Now prod's issuer is + # keycloak.prod.home.keithjackson.dev, and the public record for that name is published by + # PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer + # and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed. + # + # STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's + # external-dns has actually published the record. Until then the issuer does not resolve + # from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass + # path, and it always works. Ordering, not a design gap. node_baseline_dns_search: [] +# --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) -------------------- +# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/ +# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and +# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a +# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so +# postBuild.substituteFrom never sees it. Full rationale in +# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this. +# +# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the +# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns +# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE); +# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is +# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client +# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against. +k3s_base_domain: "prod.home.keithjackson.dev" + # --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) ----- # roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE # and group_vars/all.yml carries no override, so dev keeps resolving exactly that). diff --git a/infra/ansible/roles/cilium/defaults/main.yml b/infra/ansible/roles/cilium/defaults/main.yml index c22b05a..f5f70eb 100644 --- a/infra/ansible/roles/cilium/defaults/main.yml +++ b/infra/ansible/roles/cilium/defaults/main.yml @@ -15,9 +15,20 @@ cilium_namespace: "kube-system" # Cilium 1.20.1 targets Gateway API v1.6.1. gateway_api_version: "v1.6.1" gateway_api_crd_urls: + # ONE URL, deliberately. As of v1.6.0 the STANDARD channel bundle already contains + # TLSRoute (plus TCPRoute, UDPRoute, ListenerSet and BackendTLSPolicy), so the separate + # experimental TLSRoute manifest this list used to carry is redundant — and worse than + # redundant: v1.6 also ships a ValidatingAdmissionPolicy, + # `safe-upgrades.gateway.networking.k8s.io`, which REJECTS an experimental-channel CRD + # applied over a standard-channel one. Applying the old second URL therefore fails hard: + # ValidatingAdmissionPolicy 'safe-upgrades.gateway.networking.k8s.io' ... denied request: + # Installing experimental CRDs on top of standard channel CRDs is prohibited by default. + # Verified 2026-09-04 on the prod bootstrap: standard-install.yaml v1.6.1 alone produces the + # exact 10 gateway.networking.k8s.io CRDs the LIVE dev cluster is running, tlsroutes included + # and annotated channel=standard, bundle-version=v1.6.1 — so this is parity with dev, not a + # reduction. Dev needs no reapply; it already has them. If gateway_api_version is ever moved + # BACK below v1.6.0, the experimental TLSRoute URL has to come back with it. - "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{ gateway_api_version }}/standard-install.yaml" - # TLSRoute (experimental) is required by Cilium's Gateway API support. - - "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{ gateway_api_version }}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" # Local kubeconfig fetched from the primary. *.kubeconfig is gitignored. The FILENAME # comes from cluster_kubeconfig_name (group_vars) so a prod run cannot overwrite the dev diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml index 4875e1d..d052d12 100644 --- a/infra/ansible/roles/k3s_server/defaults/main.yml +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -66,3 +66,33 @@ k3s_tls_sans: >- {{ ([k3s_api_host, kube_vip_address, '127.0.0.1'] + (groups[k3s_servers_group] | map('extract', hostvars, 'k3s_node_ip') | list)) | unique }} + +# --- Keycloak OIDC issuer for kubectl SSO (R-IDP-03, R-ENV-02) ----------------------- +# THE ANSIBLE-SIDE HALF OF ${BASE_DOMAIN}, and it cannot be the Flux-side half. +# +# The kube-apiserver's `oidc-issuer-url` is a k3s SERVER FLAG written into +# /etc/rancher/k3s/config.yaml by this role — it is NOT a Kubernetes manifest, so Flux never +# sees it and `postBuild.substituteFrom: cluster-vars` cannot reach it. The platform hostname +# suffix therefore exists as ONE FACT IN TWO PLACES, and they must agree: +# +# Flux side clusters/base/cluster-vars.yaml BASE_DOMAIN (per-overlay) +# Ansible side THIS variable k3s_base_domain (per-inventory-group) +# +# [parity] k3s_base_domain MUST equal the BASE_DOMAIN the SAME cluster's Flux overlay sets. +# If they disagree, the apiserver validates ID tokens against a DIFFERENT Keycloak than the +# one the cluster actually runs: every `kubectl` SSO login fails with an issuer mismatch, +# while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken", +# not like a config skew. The break-glass cert kubeconfig is why this is recoverable. +# +# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same +# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the +# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte +# what it already is and the Restart handler does NOT fire on a re-run. +# +# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam +# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it +# from group membership rather than from anyone remembering. +k3s_base_domain: "home.keithjackson.dev" + +# Derived, so the realm path is stated once. Override k3s_base_domain, not this. +k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab" diff --git a/infra/ansible/roles/k3s_server/tasks/main.yml b/infra/ansible/roles/k3s_server/tasks/main.yml index f925742..232f3cf 100644 --- a/infra/ansible/roles/k3s_server/tasks/main.yml +++ b/infra/ansible/roles/k3s_server/tasks/main.yml @@ -75,13 +75,19 @@ # OIDC (Keycloak SSO for kubectl). The API server trusts ID tokens from the homelab realm; # the `kubernetes` PUBLIC client (PKCE) + kubelogin mint them, and RBAC binds the `groups` # claim (platform/keycloak/k8s-oidc-rbac.yaml: oidc:platform-admins->cluster-admin, - # oidc:viewers->view). Nodes resolve keycloak.home.keithjackson.dev to the Cilium gateway VIP - # (192.168.1.53) and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:` + # oidc:viewers->view). Nodes resolve keycloak. to the Cilium gateway VIP + # and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:` # prefixes namespace federated identities away from cert / service-account subjects. # ADDITIVE + break-glass-safe: the cert-based admin kubeconfig keeps full access with or # without Keycloak, so a bad realm/OIDC state can never lock you out of the cluster. + # + # NOT a Flux ${BASE_DOMAIN} token: this is a k3s SERVER FLAG, not a manifest, so + # postBuild.substituteFrom never touches it. k3s_oidc_issuer_url derives from + # k3s_base_domain (defaults/main.yml), whose [parity] contract with the cluster's + # BASE_DOMAIN is documented there. Dev's default renders the value it already runs, so + # config.yaml is unchanged and the Restart handler does not fire. kube-apiserver-arg: - - "oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab" + - "oidc-issuer-url={{ k3s_oidc_issuer_url }}" - "oidc-client-id=kubernetes" - "oidc-username-claim=email" - "oidc-username-prefix=oidc:" @@ -98,15 +104,20 @@ INSTALL_K3S_VERSION: "{{ k3s_version }}" when: k3s_server_mode == 'primary' +# PATH FOLLOWS --data-dir. k3s writes the node-token to /server/token, NOT to +# the literal /var/lib/rancher/k3s. On dev k3s_data_dir IS /var/lib/rancher/k3s (the role +# default, restated explicitly above) so this is byte-for-byte the old path; on prod +# --data-dir is /data/k3s (group_vars/k3s_prod) and the hardcoded path did not exist at all +# — this task would have timed out after 180s and the two secondaries could never join. - name: Wait for the k3s node-token to exist (primary) ansible.builtin.wait_for: - path: /var/lib/rancher/k3s/server/token + path: "{{ k3s_data_dir }}/server/token" timeout: 180 when: k3s_server_mode == 'primary' - name: Read the k3s node-token (primary) ansible.builtin.slurp: - src: /var/lib/rancher/k3s/server/token + src: "{{ k3s_data_dir }}/server/token" register: k3s_token_raw when: k3s_server_mode == 'primary' @@ -126,9 +137,11 @@ # "The primary" is resolved through k3s_primary_group (group_vars: `k3s_primary` for # dev, `k3s_prod_primary` for prod), never a literal group name — so a prod secondary # delegates to the PROD primary using this same role, unchanged (R-ENV-02). +# k3s_data_dir is a CLUSTER-WIDE value (group_vars for prod, role default for dev), so the +# secondary's own copy of it is also the primary's data dir — correct for the delegated read. - name: Fetch the k3s node-token directly from the primary (secondary) ansible.builtin.slurp: - src: /var/lib/rancher/k3s/server/token + src: "{{ k3s_data_dir }}/server/token" delegate_to: "{{ groups[k3s_primary_group][0] }}" register: k3s_primary_token_raw when: diff --git a/infra/ansible/roles/kube_vip/defaults/main.yml b/infra/ansible/roles/kube_vip/defaults/main.yml index 9718a4e..9d8e677 100644 --- a/infra/ansible/roles/kube_vip/defaults/main.yml +++ b/infra/ansible/roles/kube_vip/defaults/main.yml @@ -31,4 +31,14 @@ kube_vip_retry_period: 2 # k3s auto-deploy manifests dir: k3s applies anything here on start, so kube-vip # is reproducible and independent of Flux/Helm. -kube_vip_manifest_path: "/var/lib/rancher/k3s/server/manifests/kube-vip.yaml" +# +# THE PATH FOLLOWS --data-dir. k3s reads auto-deploy manifests from +# /server/manifests, so this must be derived from the cluster-wide +# k3s_data_dir (roles/k3s_server turns the same variable into --data-dir), not from the +# literal /var/lib/rancher/k3s. Dev leaves k3s_data_dir unset outside roles/k3s_server's +# own defaults, so the fallback below reproduces dev's existing path exactly; prod sets +# k3s_data_dir: /data/k3s in group_vars/k3s_prod, where the old hardcoded path was simply +# not the directory k3s watches — the manifest would have been written and silently never +# applied, and the control-plane VIP would never have come up. +kube_vip_manifest_path: >- + {{ k3s_data_dir | default('/var/lib/rancher/k3s') }}/server/manifests/kube-vip.yaml diff --git a/infra/ansible/roles/kube_vip/tasks/main.yml b/infra/ansible/roles/kube_vip/tasks/main.yml index db63d7e..3312477 100644 --- a/infra/ansible/roles/kube_vip/tasks/main.yml +++ b/infra/ansible/roles/kube_vip/tasks/main.yml @@ -1,13 +1,22 @@ --- # Deploy kube-vip (control-plane HA VIP) via the k3s auto-deploy manifests dir. -# k3s applies /var/lib/rancher/k3s/server/manifests/*.yaml on start, so this is -# reproducible and independent of Flux. The same manifest is written on every +# k3s applies /server/manifests/*.yaml on start, so this is +# reproducible and independent of Flux (see kube_vip_manifest_path in defaults/main.yml — +# the dir follows k3s's --data-dir, which prod moves to /data/k3s). The same manifest is written on every # server so the DaemonSet survives the loss of any single node. -- name: Detect the LAN interface holding the node's 192.168.1.5x address +# WHICH INTERFACE kube-vip ARPs THE VIP ON. Detected from this node's OWN k3s_node_ip +# (inventory, per host) rather than a hardcoded subnet: the previous awk matched the +# literal 192.168.1.5[0-2], i.e. DEV's node addresses, so on prod (192.168.2.10/.11/.12) +# it returned nothing and the assert below aborted the play. k3s_node_ip is the same +# address k3s advertises with --node-ip and is defined for dev and prod alike, so this +# resolves eno1 on prod and the identical dev interface it always resolved before. +# `ip addr show to /32` does the matching in iproute2 instead of a regex over dotted +# quads (where `.` would match any character). +- name: Detect the LAN interface holding this node's k3s_node_ip ansible.builtin.shell: >- set -o pipefail; - ip -o -4 addr show | awk '/192\.168\.1\.5[0-2]\//{print $2; exit}' + ip -o -4 addr show to {{ k3s_node_ip }}/32 | awk '{print $2; exit}' args: executable: /bin/bash register: kube_vip_iface_detect @@ -25,7 +34,7 @@ - kube_vip_vip_interface is defined - kube_vip_vip_interface | length > 0 fail_msg: >- - Could not detect the 192.168.1.5x LAN interface for kube-vip on + Could not detect the interface holding {{ k3s_node_ip }} for kube-vip on {{ inventory_hostname }} — set kube_vip_interface explicitly. - name: Ensure the k3s auto-deploy manifests directory exists diff --git a/platform/coredns/coredns-custom.yaml b/platform/coredns/coredns-custom.yaml index 189307e..4e2e0a5 100644 --- a/platform/coredns/coredns-custom.yaml +++ b/platform/coredns/coredns-custom.yaml @@ -29,9 +29,9 @@ metadata: data: # Filename `home.server` -> imported as a standalone server block for the whole zone. home.server: | - home.keithjackson.dev:53 { + ${BASE_DOMAIN}:53 { errors - # Answer EVERY name in the zone (apex + all `.home.keithjackson.dev`) with the Gateway + # Answer EVERY name in the zone (apex + all `.${BASE_DOMAIN}`) with the Gateway # VIP. The template plugin's default match is `.*`, so this covers current and FUTURE app # hostnames without editing this file per service. `{{ .Name }}` echoes the queried FQDN. template IN A { diff --git a/platform/crafty/deployment.yaml b/platform/crafty/deployment.yaml index 8bcd53d..6c1b7d6 100644 --- a/platform/crafty/deployment.yaml +++ b/platform/crafty/deployment.yaml @@ -125,7 +125,7 @@ spec: - --http-address=0.0.0.0:4180 - --reverse-proxy=true - --provider=oidc - - --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab + - --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab # Gate the co-located Crafty UI. Crafty serves HTTPS with a SELF-SIGNED cert, so skip verify # on this in-pod (127.0.0.1) hop; the browser-facing TLS is the trusted wildcard at the Gateway. - --upstream=https://127.0.0.1:8443 @@ -138,9 +138,9 @@ spec: - --allowed-group=platform-admins # Single provider => 302 straight to Keycloak, no oauth2-proxy interstitial button. - --skip-provider-button=true - - --redirect-url=https://crafty.home.keithjackson.dev/oauth2/callback - - --cookie-domain=.home.keithjackson.dev - - --whitelist-domain=.home.keithjackson.dev + - --redirect-url=https://crafty.${BASE_DOMAIN}/oauth2/callback + - --cookie-domain=.${BASE_DOMAIN} + - --whitelist-domain=.${BASE_DOMAIN} - --cookie-secure=true - --pass-access-token=true - --set-xauthrequest=true diff --git a/platform/crafty/httproute.yaml b/platform/crafty/httproute.yaml index 6324339..b79ff1b 100644 --- a/platform/crafty/httproute.yaml +++ b/platform/crafty/httproute.yaml @@ -25,7 +25,7 @@ spec: namespace: gateway sectionName: https hostnames: - - crafty.home.keithjackson.dev + - crafty.${BASE_DOMAIN} rules: - matches: - path: diff --git a/platform/crafty/service-l4.yaml b/platform/crafty/service-l4.yaml index 0501f7c..97c9039 100644 --- a/platform/crafty/service-l4.yaml +++ b/platform/crafty/service-l4.yaml @@ -42,7 +42,7 @@ metadata: lbipam.cilium.io/sharing-key: "crafty" lbipam.cilium.io/ips: "${CRAFTY_L4_VIP}" # external-dns publishes A crafty-mc.home.keithjackson.dev -> the pinned VIP (Java server target). - external-dns.alpha.kubernetes.io/hostname: "crafty-mc.home.keithjackson.dev" + external-dns.alpha.kubernetes.io/hostname: "crafty-mc.${BASE_DOMAIN}" spec: type: LoadBalancer externalTrafficPolicy: Local diff --git a/platform/externaldns/helmrelease.yaml b/platform/externaldns/helmrelease.yaml index 01057c7..3a56c3b 100644 --- a/platform/externaldns/helmrelease.yaml +++ b/platform/externaldns/helmrelease.yaml @@ -29,15 +29,37 @@ spec: sources: - gateway-httproute - service - # domain-filter MUST be the actual Cloudflare ZONE (keithjackson.dev), not the platform subdomain - # (home.keithjackson.dev) — ExternalDNS filters the account's zones by this value, and a subdomain - # filter drops the parent zone → "no hosted zone matching record DNS Name" and nothing is created. - # Scope stays effectively home.keithjackson.dev because that's the only suffix services request; - # policy=sync + registry=txt only ever touch ExternalDNS-OWNED records, never the pre-existing - # wildcard or other keithjackson.dev records. (docs/09; resolves the WS-NET TODO below.) + # --------------------------------------------------------------------------------------------- + # CROSS-CLUSTER DNS OWNERSHIP. Read before changing either knob below. + # + # On 2026-09-04 prod's external-dns DELETED EVERY ONE of dev's public records. Both clusters ran + # `policy: sync` against the SAME Cloudflare zone, served the SAME hostnames, and — because + # NEITHER set txtOwnerId — both inherited the chart default owner id `default`. Prod therefore + # read dev's TXT registry entries as its own, found no local resource claiming those names, and + # garbage-collected them. `policy: sync` is correct and stays; what was missing was identity. + # Full write-up: infra/PROD-PROVISIONING.md §10. + # + # TWO INDEPENDENT GUARDS NOW, either of which alone prevents a recurrence: + # 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns. + # 2. domainFilters — per-environment name scope; prod cannot even see names outside its + # own subtree. Defence in depth, added after the incident. + # --------------------------------------------------------------------------------------------- + # Which names this instance may manage at all. Cloudflare-provider caveat, empirically hit + # 2026-08: this value is used for ZONE selection as well as record matching, and a filter that is + # a strict subdomain of the zone dropped the parent zone entirely ("no hosted zone matching record + # DNS Name") — so DEV keeps the zone apex `keithjackson.dev`, which is also what dev serves the + # apex of. PROD narrows to `prod.home.keithjackson.dev`. If prod's first reconcile logs no + # matching zone, that caveat still applies: prod creates NOTHING (fail-closed, never destructive) + # and the fallback is to widen prod's EXTERNALDNS_DOMAIN_FILTER back to the zone apex, leaving + # txtOwnerId as the sole guard. See clusters/base/cluster-vars.yaml. domainFilters: - - "${DNS_ZONE}" - policy: sync # create + update + delete records as services come and go + - "${EXTERNALDNS_DOMAIN_FILTER}" + # The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence + # caused the incident. Dev renders `default` (its live records are already registered under that + # owner; changing it would orphan them and duplicate every record). Prod renders `prod`. + txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}" + policy: sync # create + update + delete records as services come and go — but ONLY + # records this owner id owns AND this domainFilter admits (see above). # Cloudflare API token: SOPS-encrypted secret, referenced BY NAME only (OQ-4, ADR-0007). env: - name: CF_API_TOKEN diff --git a/platform/gateway/gateway.yaml b/platform/gateway/gateway.yaml index 5e9ba65..d4f0589 100644 --- a/platform/gateway/gateway.yaml +++ b/platform/gateway/gateway.yaml @@ -13,7 +13,7 @@ spec: - name: https protocol: HTTPS port: 443 - hostname: "*.home.keithjackson.dev" + hostname: "*.${BASE_DOMAIN}" tls: mode: Terminate certificateRefs: @@ -30,7 +30,7 @@ spec: - name: http-redirect protocol: HTTP port: 80 - hostname: "*.home.keithjackson.dev" + hostname: "*.${BASE_DOMAIN}" allowedRoutes: namespaces: from: Selector @@ -46,7 +46,7 @@ spec: - name: https-apex protocol: HTTPS port: 443 - hostname: "home.keithjackson.dev" + hostname: "${BASE_DOMAIN}" tls: mode: Terminate certificateRefs: @@ -61,7 +61,7 @@ spec: - name: http-apex-redirect protocol: HTTP port: 80 - hostname: "home.keithjackson.dev" + hostname: "${BASE_DOMAIN}" allowedRoutes: namespaces: from: Selector diff --git a/platform/gateway/wildcard-certificate.yaml b/platform/gateway/wildcard-certificate.yaml index 12a5bbe..97dfb6f 100644 --- a/platform/gateway/wildcard-certificate.yaml +++ b/platform/gateway/wildcard-certificate.yaml @@ -10,8 +10,8 @@ metadata: spec: secretName: wildcard-home-tls # consumed by the Gateway HTTPS listener below dnsNames: - - "*.home.keithjackson.dev" - - "home.keithjackson.dev" # apex for the C2 UI (docs/09) + - "*.${BASE_DOMAIN}" + - "${BASE_DOMAIN}" # apex for the C2 UI (docs/09) issuerRef: kind: ClusterIssuer # ${ACME_CLUSTER_ISSUER}: dev -> letsencrypt-staging, prod -> letsencrypt-prod (cluster-vars). diff --git a/platform/headlamp/httproute.yaml b/platform/headlamp/httproute.yaml index 433e6bd..eaccb53 100644 --- a/platform/headlamp/httproute.yaml +++ b/platform/headlamp/httproute.yaml @@ -24,7 +24,7 @@ spec: namespace: gateway sectionName: https hostnames: - - headlamp.home.keithjackson.dev + - headlamp.${BASE_DOMAIN} rules: - matches: - path: diff --git a/platform/homepage/configmap.yaml b/platform/homepage/configmap.yaml index 0bec97e..8819a10 100644 --- a/platform/homepage/configmap.yaml +++ b/platform/homepage/configmap.yaml @@ -36,63 +36,63 @@ data: --- - Platform: - Headlamp: - href: https://headlamp.home.keithjackson.dev + href: https://headlamp.${BASE_DOMAIN} description: Kubernetes dashboard (per-user OIDC RBAC) icon: mdi-kubernetes - Grafana: - href: https://grafana.home.keithjackson.dev + href: https://grafana.${BASE_DOMAIN} description: Metrics, logs & dashboards (fronts Prometheus/Loki/Alertmanager) icon: mdi-chart-areaspline - Semaphore: - href: https://semaphore.home.keithjackson.dev + href: https://semaphore.${BASE_DOMAIN} description: Ansible automation & CI icon: mdi-cog-play - Gitea: - href: https://gitea.home.keithjackson.dev + href: https://gitea.${BASE_DOMAIN} description: Git hosting & Actions (GitOps source of truth) icon: mdi-git - Keycloak: - href: https://keycloak.home.keithjackson.dev + href: https://keycloak.${BASE_DOMAIN} description: Identity provider (SSO / OIDC) icon: mdi-account-key - Media: - Plex: - href: https://plex.home.keithjackson.dev + href: https://plex.${BASE_DOMAIN} description: Media streaming icon: mdi-plex - Seerr: - href: https://seerr.home.keithjackson.dev + href: https://seerr.${BASE_DOMAIN} description: Media requests icon: mdi-play-box-multiple - Sonarr: - href: https://sonarr.home.keithjackson.dev + href: https://sonarr.${BASE_DOMAIN} description: TV library manager icon: mdi-television-classic - Radarr: - href: https://radarr.home.keithjackson.dev + href: https://radarr.${BASE_DOMAIN} description: Movie library manager icon: mdi-movie-open - Jackett: - href: https://jackett.home.keithjackson.dev + href: https://jackett.${BASE_DOMAIN} description: Indexer proxy icon: mdi-magnify - qBittorrent: - href: https://qbittorrent.home.keithjackson.dev + href: https://qbittorrent.${BASE_DOMAIN} description: Download client (VPN egress) icon: mdi-download-network - Network & Infra: - Pi-hole: - href: https://pihole.home.keithjackson.dev + href: https://pihole.${BASE_DOMAIN} description: Network-wide DNS & ad-blocking icon: mdi-shield-check - UniFi: - href: https://unifi.home.keithjackson.dev:11443 + href: https://unifi.${BASE_DOMAIN}:11443 description: Network controller (UOS) icon: mdi-router-network - Nextcloud: - href: https://nextcloud.home.keithjackson.dev + href: https://nextcloud.${BASE_DOMAIN} description: Files, calendar & contacts icon: mdi-cloud @@ -104,7 +104,7 @@ data: href: https://gitea.keithjackson.dev - Auth portal: - abbr: SSO - href: https://auth.home.keithjackson.dev + href: https://auth.${BASE_DOMAIN} - Reference: - Homepage docs: - abbr: HP diff --git a/platform/homepage/deployment.yaml b/platform/homepage/deployment.yaml index c8bbce4..6ad6ce6 100644 --- a/platform/homepage/deployment.yaml +++ b/platform/homepage/deployment.yaml @@ -54,7 +54,7 @@ spec: # REQUIRED by Homepage >= 1.0 — requests whose Host header is not in this list get 403. # The public apex host (served via the Gateway) + the pod IP (for kubelet probes). - name: HOMEPAGE_ALLOWED_HOSTS - value: "home.keithjackson.dev,$(MY_POD_IP):3000" + value: "${BASE_DOMAIN},$(MY_POD_IP):3000" resources: requests: cpu: 25m diff --git a/platform/homepage/httproute.yaml b/platform/homepage/httproute.yaml index 3b64a53..cb17b70 100644 --- a/platform/homepage/httproute.yaml +++ b/platform/homepage/httproute.yaml @@ -28,7 +28,7 @@ spec: namespace: gateway sectionName: https-apex # the apex listener (wildcard `https` won't match the bare apex) hostnames: - - home.keithjackson.dev + - ${BASE_DOMAIN} rules: - matches: - path: diff --git a/platform/keycloak/httproute.yaml b/platform/keycloak/httproute.yaml index 3595c92..34dae64 100644 --- a/platform/keycloak/httproute.yaml +++ b/platform/keycloak/httproute.yaml @@ -16,7 +16,7 @@ spec: namespace: gateway sectionName: https hostnames: - - keycloak.home.keithjackson.dev + - keycloak.${BASE_DOMAIN} rules: - matches: - path: diff --git a/platform/keycloak/keycloak-instance.yaml b/platform/keycloak/keycloak-instance.yaml index a86e9f5..4969995 100644 --- a/platform/keycloak/keycloak-instance.yaml +++ b/platform/keycloak/keycloak-instance.yaml @@ -30,7 +30,7 @@ spec: poolMaxSize: 20 # --- Exposure: TLS at the Gateway; Keycloak serves HTTP and trusts the proxy headers. --- hostname: - hostname: https://keycloak.home.keithjackson.dev + hostname: https://keycloak.${BASE_DOMAIN} http: httpEnabled: true # TLS is terminated at the Gateway (edge), not in Keycloak proxy: diff --git a/platform/keycloak/keycloak-realm-import.yaml b/platform/keycloak/keycloak-realm-import.yaml index b637814..cba2312 100644 --- a/platform/keycloak/keycloak-realm-import.yaml +++ b/platform/keycloak/keycloak-realm-import.yaml @@ -98,9 +98,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:GRAFANA_CLIENT_SECRET)" redirectUris: - - https://grafana.home.keithjackson.dev/login/generic_oauth + - https://grafana.${BASE_DOMAIN}/login/generic_oauth webOrigins: - - https://grafana.home.keithjackson.dev + - https://grafana.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -125,10 +125,10 @@ spec: directAccessGrantsEnabled: false secret: "$(env:OAUTH2_PROXY_CLIENT_SECRET)" redirectUris: - - https://*.home.keithjackson.dev/oauth2/callback - - https://auth.home.keithjackson.dev/oauth2/callback + - https://*.${BASE_DOMAIN}/oauth2/callback + - https://auth.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://*.home.keithjackson.dev + - https://*.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -209,9 +209,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:GITEA_CLIENT_SECRET)" redirectUris: - - https://gitea.home.keithjackson.dev/user/oauth2/keycloak/callback + - https://gitea.${BASE_DOMAIN}/user/oauth2/keycloak/callback webOrigins: - - https://gitea.home.keithjackson.dev + - https://gitea.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -240,9 +240,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:NEXTCLOUD_CLIENT_SECRET)" redirectUris: - - https://nextcloud.home.keithjackson.dev/apps/user_oidc/code + - https://nextcloud.${BASE_DOMAIN}/apps/user_oidc/code webOrigins: - - https://nextcloud.home.keithjackson.dev + - https://nextcloud.${BASE_DOMAIN} # Post-logout redirect allow-list (#logout "redirect failed"). On sign-out user_oidc redirects the # browser to Keycloak's end_session_endpoint with post_logout_redirect_uri= and # client_id=nextcloud but NO id_token_hint (provider sendIdTokenHint=0), so Keycloak validates the @@ -250,7 +250,7 @@ spec: # "redirect failed". The wildcard covers the base URL and any logout landing path. Keycloak stores # this as the client attribute `post.logout.redirect.uris` (## separates multiple values). attributes: - post.logout.redirect.uris: https://nextcloud.home.keithjackson.dev/* + post.logout.redirect.uris: https://nextcloud.${BASE_DOMAIN}/* defaultClientScopes: - email - profile @@ -279,9 +279,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:SEMAPHORE_CLIENT_SECRET)" redirectUris: - - https://semaphore.home.keithjackson.dev/api/auth/oidc/keycloak/redirect + - https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect webOrigins: - - https://semaphore.home.keithjackson.dev + - https://semaphore.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -318,9 +318,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:HEADLAMP_CLIENT_SECRET)" redirectUris: - - https://headlamp.home.keithjackson.dev/oidc-callback + - https://headlamp.${BASE_DOMAIN}/oidc-callback webOrigins: - - https://headlamp.home.keithjackson.dev + - https://headlamp.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -363,9 +363,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:CRAFTY_CLIENT_SECRET)" redirectUris: - - https://crafty.home.keithjackson.dev/oauth2/callback + - https://crafty.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://crafty.home.keithjackson.dev + - https://crafty.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -398,9 +398,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:PIHOLE_CLIENT_SECRET)" redirectUris: - - https://pihole.home.keithjackson.dev/oauth2/callback + - https://pihole.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://pihole.home.keithjackson.dev + - https://pihole.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -433,9 +433,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:JACKETT_CLIENT_SECRET)" redirectUris: - - https://jackett.home.keithjackson.dev/oauth2/callback + - https://jackett.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://jackett.home.keithjackson.dev + - https://jackett.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -459,9 +459,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:QBITTORRENT_CLIENT_SECRET)" redirectUris: - - https://qbittorrent.home.keithjackson.dev/oauth2/callback + - https://qbittorrent.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://qbittorrent.home.keithjackson.dev + - https://qbittorrent.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -485,9 +485,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:RADARR_CLIENT_SECRET)" redirectUris: - - https://radarr.home.keithjackson.dev/oauth2/callback + - https://radarr.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://radarr.home.keithjackson.dev + - https://radarr.${BASE_DOMAIN} defaultClientScopes: - email - profile @@ -511,9 +511,9 @@ spec: directAccessGrantsEnabled: false secret: "$(env:SONARR_CLIENT_SECRET)" redirectUris: - - https://sonarr.home.keithjackson.dev/oauth2/callback + - https://sonarr.${BASE_DOMAIN}/oauth2/callback webOrigins: - - https://sonarr.home.keithjackson.dev + - https://sonarr.${BASE_DOMAIN} defaultClientScopes: - email - profile diff --git a/platform/kubevirt/alerts.yaml b/platform/kubevirt/alerts.yaml index fbaff98..03e9ac0 100644 --- a/platform/kubevirt/alerts.yaml +++ b/platform/kubevirt/alerts.yaml @@ -30,7 +30,7 @@ spec: annotations: summary: "KubeVirt VM(s) failing to launch in {{ $labels.namespace }}" description: "One or more VirtualMachineInstances are in the Failed phase in namespace {{ $labels.namespace }} for 5m — a failed VM launch (image import, scheduling, or qemu error). Check `kubectl get vmi -n {{ $labels.namespace }}` and the virt-launcher pod events." - runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-launch-failed" + runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-launch-failed" # Crash-loop: the VM's virt-launcher pod is restarting repeatedly (guest kernel panic, OOM, or a # crashing qemu). >3 restarts in 15m is a crash-loop, not a one-off reboot. - alert: KubeVirtVMCrashLooping @@ -42,7 +42,7 @@ spec: annotations: summary: "KubeVirt VM crash-looping: {{ $labels.namespace }}/{{ $labels.pod }}" description: "virt-launcher pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in 15m — the guest VM is crash-looping (panic / OOM / qemu crash)." - runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-crashloop" + runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-crashloop" # Control plane down: no virt-controller or virt-api replica reports ready, so no VM lifecycle # operations can proceed. The metrics are `kubevirt_virt_{controller,api}_ready_status` (per-pod # 1=ready); sum()==0 means every replica is unready. The absent() clause also fires if the whole @@ -59,7 +59,7 @@ spec: annotations: summary: "KubeVirt control plane degraded" description: "The KubeVirt control plane (virt-controller/virt-api) is not ready or its metrics are absent for 10m — VM lifecycle operations (create/start/stop) will fail (docs/06)." - runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-control-plane" + runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-control-plane" # CDI import stuck: a DataVolume import/clone still in flight after 30m blocks the VM it backs. # kubevirt_cdi_import_pods_high_restart counts importer pods stuck restarting; a positive value # sustained means an import is not progressing (metric name confirmed against CDI at bring-up). @@ -71,4 +71,4 @@ spec: annotations: summary: "CDI DataVolume import stuck: {{ $labels.namespace }}/{{ $labels.name }}" description: "A CDI DataVolume import/clone in {{ $labels.namespace }} has not completed after 30m — the VM disk it backs cannot start. Check the importer pod (often an egress/default-deny block on the image source, or a slow mirror)." - runbook_url: "https://home.keithjackson.dev/runbooks/cdi-import-stuck" + runbook_url: "https://${BASE_DOMAIN}/runbooks/cdi-import-stuck" diff --git a/platform/monitoring/alerts.yaml b/platform/monitoring/alerts.yaml index f6d7ed9..63939f9 100644 --- a/platform/monitoring/alerts.yaml +++ b/platform/monitoring/alerts.yaml @@ -38,7 +38,7 @@ spec: annotations: summary: "Node exporter down on {{ $labels.instance }}" description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable." - runbook_url: "https://home.keithjackson.dev/runbooks/node-down" + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-down" - alert: NodeMemoryPressureHigh expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90 for: 10m @@ -47,7 +47,7 @@ spec: annotations: summary: "Memory pressure high on {{ $labels.instance }}" description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure." - runbook_url: "https://home.keithjackson.dev/runbooks/node-memory" + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-memory" - alert: NodeDiskFillingUp expr: | (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} @@ -58,7 +58,7 @@ spec: annotations: summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})" description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m." - runbook_url: "https://home.keithjackson.dev/runbooks/node-disk" + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-disk" - alert: NodeCPUPressureHigh # PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to # run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU @@ -71,7 +71,7 @@ spec: annotations: summary: "CPU pressure high on {{ $labels.instance }}" description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads." - runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu" + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu" - alert: NodeCPULoadHigh # 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more # runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node. @@ -86,7 +86,7 @@ spec: annotations: summary: "Load per core high on {{ $labels.instance }}" description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count." - runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu" + runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu" # ===================================================================================== # Storage — Longhorn @@ -113,7 +113,7 @@ spec: annotations: summary: "Longhorn volume degraded: {{ $labels.volume }}" description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m." - runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-degraded" + runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-degraded" - alert: LonghornVolumeFaulted expr: longhorn_volume_robustness == 3 for: 1m @@ -122,7 +122,7 @@ spec: annotations: summary: "Longhorn volume FAULTED: {{ $labels.volume }}" description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required." - runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-faulted" + runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-faulted" - alert: LonghornVolumeCapacityHigh expr: (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85 for: 30m @@ -131,7 +131,7 @@ spec: annotations: summary: "Longhorn volume near capacity: {{ $labels.volume }}" description: "Volume {{ $labels.volume }} is over 85% used for 30m." - runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity" + runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity" - alert: LonghornNodeStorageHigh expr: (longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes) > 0.85 for: 30m @@ -140,7 +140,7 @@ spec: annotations: summary: "Longhorn node storage near capacity: {{ $labels.node }}" description: "Longhorn node {{ $labels.node }} is over 85% of its storage capacity for 30m." - runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity" + runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity" # ===================================================================================== # GitOps — Flux @@ -169,7 +169,7 @@ spec: annotations: summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)." - runbook_url: "https://home.keithjackson.dev/runbooks/flux-reconcile" + runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-reconcile" - alert: FluxResourceSuspended expr: | max by (namespace, name, kind) (gotk_suspend_status) == 1 @@ -179,7 +179,7 @@ spec: annotations: summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off." - runbook_url: "https://home.keithjackson.dev/runbooks/flux-suspended" + runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-suspended" # ===================================================================================== # Certificates — cert-manager @@ -205,7 +205,7 @@ spec: annotations: summary: "Certificate expiring soon: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days." - runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry" + runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry" - alert: CertificateExpiryImminent expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600) for: 1h @@ -214,7 +214,7 @@ spec: annotations: summary: "Certificate expiry imminent: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed." - runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry" + runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry" - alert: CertificateNotReady expr: certmanager_certificate_ready_status{condition="False"} == 1 for: 1h @@ -223,7 +223,7 @@ spec: annotations: summary: "Certificate not ready / issuance failing: {{ $labels.name }}" description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)." - runbook_url: "https://home.keithjackson.dev/runbooks/cert-issuance" + runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-issuance" # ===================================================================================== # Networking — VPN / egress-gateway / denied flows (Hubble) @@ -255,7 +255,7 @@ spec: annotations: summary: "Egress VPN tunnel down" description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)." - runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down" + runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-down" # (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun # retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd # service, steering is Cilium eBPF). Tunnel + gateway-node health is covered by `VPNTunnelDown` @@ -270,7 +270,7 @@ spec: annotations: summary: "Spike in policy-denied network flows" description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)." - runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows" + runbook_url: "https://${BASE_DOMAIN}/runbooks/denied-flows" # ===================================================================================== # Backups — Velero @@ -296,7 +296,7 @@ spec: annotations: summary: "Velero backup failed: schedule {{ $labels.schedule }}" description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)." - runbook_url: "https://home.keithjackson.dev/runbooks/backup-failure" + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure" - alert: VeleroBackupPartialFailure expr: increase(velero_backup_partial_failure_total[1h]) > 0 for: 5m @@ -305,7 +305,7 @@ spec: annotations: summary: "Velero backup partially failed" description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped." - runbook_url: "https://home.keithjackson.dev/runbooks/backup-partial" + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial" - alert: VeleroNoRecentBackup expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600) for: 1h @@ -314,7 +314,7 @@ spec: annotations: summary: "No successful Velero backup in 24h: {{ $labels.schedule }}" description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)." - runbook_url: "https://home.keithjackson.dev/runbooks/backup-missed" + runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed" # ===================================================================================== # Identity — Keycloak (SSO is a critical dependency — docs/15) @@ -345,7 +345,7 @@ spec: annotations: summary: "Keycloak IdP down" description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)." - runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down" + runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-down" - alert: KeycloakDatabaseDown # Keycloak's PostgreSQL runs as a CloudNativePG Cluster `keycloak-db` (pods keycloak-db-N). # DR-critical, Longhorn-backed. (CNPG also exports cnpg_* metrics via its PodMonitor.) @@ -358,7 +358,7 @@ spec: annotations: summary: "Keycloak database unavailable" description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)." - runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-db" + runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-db" # ===================================================================================== # The monitor's own capacity — don't let the monitor blind itself (docs/10) @@ -386,7 +386,7 @@ spec: annotations: summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}" description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind." - runbook_url: "https://home.keithjackson.dev/runbooks/monitoring-capacity" + runbook_url: "https://${BASE_DOMAIN}/runbooks/monitoring-capacity" - alert: PrometheusTargetsMissing expr: | (count(up == 0) / count(up)) > 0.10 @@ -396,7 +396,7 @@ spec: annotations: summary: "More than 10% of Prometheus targets are down" description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot." - runbook_url: "https://home.keithjackson.dev/runbooks/targets-down" + runbook_url: "https://${BASE_DOMAIN}/runbooks/targets-down" # ===================================================================================== # Automated updates — Flux image-automation + hands-off update health (docs/11, ADR-0009) @@ -430,7 +430,7 @@ spec: annotations: summary: "Flux image-update-automation failing: {{ $labels.name }}" description: "ImageUpdateAutomation {{ $labels.namespace }}/{{ $labels.name }} cannot reconcile (git write-back) for 30m — stable image advances have stopped (R-UPD-02). Check the git write credential." - runbook_url: "https://home.keithjackson.dev/runbooks/image-automation" + runbook_url: "https://${BASE_DOMAIN}/runbooks/image-automation" # An ImagePolicy/ImageRepository can't resolve the latest stable tag (registry rate-limit / # auth / bad range) → that service stops receiving updates even though automation is healthy. - alert: ImagePolicyStalled @@ -444,7 +444,7 @@ spec: annotations: summary: "Image scan/policy stalled: {{ $labels.kind }}/{{ $labels.name }}" description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} cannot resolve its stable tag for 1h (registry rate-limit / creds / range) — no longer receiving updates (R-UPD-01)." - runbook_url: "https://home.keithjackson.dev/runbooks/image-policy-stalled" + runbook_url: "https://${BASE_DOMAIN}/runbooks/image-policy-stalled" # A bad update was health-gated and AUTO-ROLLED-BACK (or is stuck mid-remediation). helm-controller # holds Ready=False while a failed upgrade remediates/rolls back to the last-good revision # (HelmRelease.spec.upgrade.remediation + .spec.rollback). This is the R-UPD-03 safety rail firing. @@ -461,4 +461,4 @@ spec: annotations: summary: "Update health-gate failed / rolled back: {{ $labels.name }}" description: "HelmRelease {{ $labels.namespace }}/{{ $labels.name }} failed its health-gated upgrade and is rolling back to the last-good revision (bad-update auto-rollback — R-UPD-03). Consider an `update.pin` escape hatch (docs/11)." - runbook_url: "https://home.keithjackson.dev/runbooks/update-rollback" + runbook_url: "https://${BASE_DOMAIN}/runbooks/update-rollback" diff --git a/platform/monitoring/grafana-httproute.yaml b/platform/monitoring/grafana-httproute.yaml index 3d7edec..e1ada5f 100644 --- a/platform/monitoring/grafana-httproute.yaml +++ b/platform/monitoring/grafana-httproute.yaml @@ -14,7 +14,7 @@ spec: namespace: gateway sectionName: https hostnames: - - grafana.home.keithjackson.dev + - grafana.${BASE_DOMAIN} rules: - matches: - path: diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index 2ed6968..67aff49 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -194,15 +194,15 @@ spec: optional: true grafana.ini: server: - root_url: https://grafana.home.keithjackson.dev + root_url: https://grafana.${BASE_DOMAIN} auth.generic_oauth: enabled: true name: Keycloak client_id: grafana scopes: "openid email profile" - auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth - token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token - api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo + auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth + token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token + api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo login_attribute_path: preferred_username email_attribute_path: email # Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins -> diff --git a/platform/oauth2-proxy/helmrelease.yaml b/platform/oauth2-proxy/helmrelease.yaml index cc9681f..5e4ce98 100644 --- a/platform/oauth2-proxy/helmrelease.yaml +++ b/platform/oauth2-proxy/helmrelease.yaml @@ -32,7 +32,7 @@ spec: existingSecret: oauth2-proxy-credentials extraArgs: provider: oidc - oidc-issuer-url: https://keycloak.home.keithjackson.dev/realms/homelab + oidc-issuer-url: https://keycloak.${BASE_DOMAIN}/realms/homelab # Forward-auth (ext-auth) mode: no upstream app; return 202 for an authenticated session so the # Gateway ext-authz can allow/deny. Auth headers are surfaced to the backend. reverse-proxy: "true" @@ -41,9 +41,9 @@ spec: pass-access-token: "true" email-domain: "*" # Single sign-in/callback host for all forward-auth-protected services on the platform zone. - redirect-url: https://auth.home.keithjackson.dev/oauth2/callback - whitelist-domain: ".home.keithjackson.dev" - cookie-domain: ".home.keithjackson.dev" + redirect-url: https://auth.${BASE_DOMAIN}/oauth2/callback + whitelist-domain: ".${BASE_DOMAIN}" + cookie-domain: ".${BASE_DOMAIN}" cookie-secure: "true" # Expose group membership so per-service group authorization (R-IDP-06) can be enforced. The # Keycloak `oauth2-proxy` client emits a `groups` claim (full.path=false => bare names like diff --git a/platform/oauth2-proxy/httproute.yaml b/platform/oauth2-proxy/httproute.yaml index b41d4ab..0978599 100644 --- a/platform/oauth2-proxy/httproute.yaml +++ b/platform/oauth2-proxy/httproute.yaml @@ -14,7 +14,7 @@ spec: namespace: gateway sectionName: https hostnames: - - auth.home.keithjackson.dev + - auth.${BASE_DOMAIN} rules: - matches: - path: diff --git a/schemas/servicedefinition.schema.json b/schemas/servicedefinition.schema.json index 0504ec9..449e980 100644 --- a/schemas/servicedefinition.schema.json +++ b/schemas/servicedefinition.schema.json @@ -9,7 +9,7 @@ "properties": { "name": { "type": "string", - "description": "DNS label -> .home.keithjackson.dev (docs/13, docs/09). Must be a valid RFC 1123 DNS label.", + "description": "DNS label -> .${BASE_DOMAIN}, i.e. .home.keithjackson.dev on dev and .prod.home.keithjackson.dev on prod (docs/13, docs/09). Must be a valid RFC 1123 DNS label.", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "minLength": 1, "maxLength": 63 @@ -223,8 +223,8 @@ "properties": { "host": { "type": "string", - "description": "FQDN under the platform zone. Must be a