Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5
|
422e1dc7e8 | ||
|
jakman85andClaude Opus 5
|
b7d6ba3d5f | ||
|
jakman85andClaude Opus 5
|
834928c237 | ||
|
jakman85andClaude Opus 5
|
c8c1d2935d | ||
|
jakman85andClaude Opus 5
|
9b8979a427 |
No files matched your search
@@ -102,7 +102,7 @@ spec:
|
||||
userData: |-
|
||||
#cloud-config
|
||||
hostname: unifi
|
||||
fqdn: unifi.home.keithjackson.dev
|
||||
fqdn: unifi.${BASE_DOMAIN}
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
packages: [qemu-guest-agent, podman, uidmap, slirp4netns, ca-certificates, curl, jq, cloud-guest-utils]
|
||||
|
||||
@@ -46,7 +46,7 @@ metadata:
|
||||
lbipam.cilium.io/ips: "${UNIFI_L4_VIP}"
|
||||
# external-dns (source: service) publishes A unifi.home.keithjackson.dev -> the pinned VIP.
|
||||
# This is the browser + device target; UI is https://unifi.home.keithjackson.dev:11443.
|
||||
external-dns.alpha.kubernetes.io/hostname: "unifi.home.keithjackson.dev"
|
||||
external-dns.alpha.kubernetes.io/hostname: "unifi.${BASE_DOMAIN}"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -90,9 +90,9 @@ spec:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN
|
||||
value: gitea.home.keithjackson.dev
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.home.keithjackson.dev/
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED
|
||||
value: 'true'
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL
|
||||
@@ -127,7 +127,7 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\
|
||||
\ break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n# A postStart hook MUST exit 0 \u2014 a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff). This hook already ends 0 on both paths, but\n# it is made EXPLICIT so the guarantee does not depend on whatever statement happens to\n# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,\n# the moment its OIDC guard was fixed and the block actually ran for the first time.\nexit 0\n"
|
||||
volumes:
|
||||
- name: data
|
||||
@@ -207,7 +207,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- gitea.home.keithjackson.dev
|
||||
- gitea.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:9117
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://jackett.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -273,7 +273,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- jackett.home.keithjackson.dev
|
||||
- jackett.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:80
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://pihole.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://pihole.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -229,7 +229,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- pihole.home.keithjackson.dev
|
||||
- pihole.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -183,7 +183,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- plex.home.keithjackson.dev
|
||||
- plex.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8080
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://qbittorrent.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -273,7 +273,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- qbittorrent.home.keithjackson.dev
|
||||
- qbittorrent.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -165,7 +165,7 @@ spec:
|
||||
exec /sbin/tini -- /opt/act/run.sh
|
||||
env:
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: https://gitea.home.keithjackson.dev
|
||||
value: https://gitea.${BASE_DOMAIN}
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -104,13 +104,13 @@ spec:
|
||||
name: nextcloud-bootstrap
|
||||
key: admin-password
|
||||
- name: NEXTCLOUD_TRUSTED_DOMAINS
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEHOST
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEPROTOCOL
|
||||
value: https
|
||||
- name: OVERWRITECLIURL
|
||||
value: https://nextcloud.home.keithjackson.dev
|
||||
value: https://nextcloud.${BASE_DOMAIN}
|
||||
- name: NEXTCLOUD_OIDC_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -125,9 +125,9 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"\
|
||||
[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"\
|
||||
$ADMIN_SRC_GROUP\"); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n"
|
||||
- "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[nextcloud-oidc]\
|
||||
\ client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"$ADMIN_SRC_GROUP\"\
|
||||
); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n"
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
@@ -228,7 +228,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- nextcloud.home.keithjackson.dev
|
||||
- nextcloud.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:7878
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://radarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -295,7 +295,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- radarr.home.keithjackson.dev
|
||||
- radarr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -110,7 +110,7 @@ spec:
|
||||
- name: SEMAPHORE_ADMIN_NAME
|
||||
value: Semaphore Admin
|
||||
- name: SEMAPHORE_ADMIN_EMAIL
|
||||
value: admin@home.keithjackson.dev
|
||||
value: admin@${BASE_DOMAIN}
|
||||
- name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -231,7 +231,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- semaphore.home.keithjackson.dev
|
||||
- semaphore.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -295,7 +295,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- sonarr.home.keithjackson.dev
|
||||
- sonarr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -223,7 +223,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- seerr.home.keithjackson.dev
|
||||
- seerr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint for the dev cluster.
|
||||
# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint. ENVIRONMENT-AGNOSTIC: which overlay
|
||||
# it applies is FLUX_ENV_PATH (default clusters/dev). roles/flux_bootstrap exports that from
|
||||
# `flux_bootstrap_env_path`, overridden to clusters/prod in group_vars/k3s_prod/vars.yml — so the
|
||||
# same script bootstraps dev and prod with no fork (R-ENV-02).
|
||||
#
|
||||
# Phase 1, T-1.4 · ADR-0004 (Flux) · ADR-0007 (SOPS+age) · R-GITOPS-01 · R-SEC-01.
|
||||
# =============================================================================
|
||||
@@ -33,7 +36,7 @@
|
||||
# 2. sops-age Secret — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
|
||||
# decryption works on first pass (../sops/age-key-secret.md). This script
|
||||
# only VERIFIES it exists; it never handles key material.
|
||||
# 3. `kubectl apply -k clusters/dev` — materialises the authored reconciliation graph:
|
||||
# 3. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
|
||||
# GitRepository `flux-system` (source) -> Kustomization `platform` ->
|
||||
# `platform-config` -> the app-wave Kustomizations `apps-wave-0/1/2`
|
||||
# (chained dependsOn) + `apps-vms`, with cluster-vars overlay
|
||||
@@ -52,7 +55,10 @@
|
||||
set -euo pipefail
|
||||
|
||||
# --- config (env-overridable) ------------------------------------------------
|
||||
FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}" # dev is the ONLY Phase-1 build target
|
||||
# Which environment overlay to apply. Dev is the DEFAULT (it was the only Phase-1 build target);
|
||||
# prod passes clusters/prod via roles/flux_bootstrap. Changing the default would silently retarget
|
||||
# every existing caller — override it, never edit it.
|
||||
FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}"
|
||||
FLUX_NAMESPACE="${FLUX_NAMESPACE:-flux-system}"
|
||||
SOPS_SECRET_NAME="${SOPS_SECRET_NAME:-sops-age}"
|
||||
# The two Phase-4 image-automation controllers are installed alongside the default set so their CRDs
|
||||
@@ -62,7 +68,7 @@ FLUX_COMPONENTS_EXTRA="${FLUX_COMPONENTS_EXTRA:-image-reflector-controller,image
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/../.." && pwd))"
|
||||
|
||||
echo "==> Flux + SOPS bootstrap (dev) — path=${FLUX_ENV_PATH}"
|
||||
echo "==> Flux + SOPS bootstrap — path=${FLUX_ENV_PATH}"
|
||||
|
||||
# --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
|
||||
for bin in flux kubectl; do
|
||||
@@ -134,7 +140,7 @@ if ! kubectl -n "${FLUX_NAMESPACE}" get secret "${SOPS_SECRET_NAME}" >/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- 3. apply the authored dev entrypoint ------------------------------------
|
||||
# --- 3. apply the authored environment entrypoint ----------------------------
|
||||
echo "==> [3/3] kubectl apply -k ${FLUX_ENV_PATH}"
|
||||
kubectl apply -k "${repo_root}/${FLUX_ENV_PATH}"
|
||||
|
||||
|
||||
@@ -1,11 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# bootstrap/gitea/bootstrap-gitea.sh — run a SINGLE Gitea container on the HOST (outside the
|
||||
# cluster) as the Phase-1 GitOps bootstrap source. While the cluster is built, Flux (in-cluster)
|
||||
# reconciles this repo from the host Gitea over the k3s-dev NAT gateway. This is the "local
|
||||
# bootstrap source" that migrates to in-cluster Gitea in Phase 6 (docs/05 "Source-of-truth",
|
||||
# CONTRIBUTING.md).
|
||||
# cluster) as the GitOps bootstrap source. While a cluster is built, Flux (in-cluster) reconciles
|
||||
# this repo from the host Gitea. This is the "local bootstrap source" that migrates to in-cluster
|
||||
# Gitea at cutover (docs/05 "Source-of-truth", CONTRIBUTING.md). Dev has already cut over; PROD is
|
||||
# using it now.
|
||||
#
|
||||
# Run by the OPERATOR at bring-up — NOT during author-only, NOT by any agent.
|
||||
# HOW THE CLUSTER REACHES THE HOST — read this before setting HOST_NAT_IP.
|
||||
# It is an ADDRESS OF THIS HOST as seen from the cluster nodes, and it differs per environment:
|
||||
#
|
||||
# PROD (bare metal, current use): HOST_NAT_IP=192.168.2.132
|
||||
# The prod nodes (192.168.2.10/.11/.12) and this workstation are on the SAME LAN. There is
|
||||
# NO NAT hop of any kind, and NO firewall change was needed — verified 2026-09-04 from all
|
||||
# three nodes. The firewalld note further down is a DEV-ERA libvirt concern; it does not
|
||||
# apply to prod, so do not "fix" prod by opening zones.
|
||||
#
|
||||
# DEV (libvirt VMs, HISTORICAL): HOST_NAT_IP=10.73.0.1 was the k3s-dev libvirt NAT gateway.
|
||||
# *** THAT NETWORK IS RETIRED. 10.73.0.1 is stale and must not be reused. *** Dev's own
|
||||
# bootstrap Gitea was later reached on 192.168.2.103:3000 and is retained STOPPED as
|
||||
# recovery (see the comment on the GitRepository patch in clusters/dev/kustomization.yaml).
|
||||
#
|
||||
# WHO RUNS THIS: the OPERATOR, or an agent the operator has EXPLICITLY DIRECTED to stand up a
|
||||
# bootstrap source for a specific bring-up (as happened 2026-09-04 for prod). It is NOT part of
|
||||
# author-only work and must never be fired off casually or "to see if it works": it starts a
|
||||
# long-lived container, generates and writes admin credentials to $HOME, creates an org and repo,
|
||||
# and PUSHES THE CURRENT HEAD to that repo's `main` — which is the revision a cluster pointed at
|
||||
# it will then reconcile. Know which HEAD you are publishing before you run it.
|
||||
set -euo pipefail
|
||||
ENGINE="${ENGINE:-podman}"
|
||||
GITEA_IMAGE="${GITEA_IMAGE:-docker.io/gitea/gitea:1.22}" # pinned; bump if needed
|
||||
@@ -16,11 +35,24 @@ SSH_PORT="${GITEA_SSH_PORT:-2222}"
|
||||
# subuid that can't write a host-owned dir, so Gitea fails to create /data/gitea/conf/app.ini.
|
||||
DATA_VOLUME="${GITEA_DATA_VOLUME:-gitea-bootstrap-data}"
|
||||
DATA_DIR="podman volume ${DATA_VOLUME}" # display label only (data lives in the named volume)
|
||||
# The cluster VMs reach the host on the k3s-dev NAT gateway (10.73.0.1). ROOT_URL must match so
|
||||
# The address the CLUSTER uses to reach this host (see the header). ROOT_URL must match it so
|
||||
# clone/redirect URLs are correct from inside the cluster.
|
||||
# NOTE: firewalld's `libvirt` zone (owns virbrN) blocks VM->host:3000 by default. Open it once:
|
||||
#
|
||||
# The DEFAULT IS DELIBERATELY UNSET — there is no correct default any more. The old default
|
||||
# (10.73.0.1) was the retired k3s-dev libvirt NAT gateway; leaving it in place would silently
|
||||
# publish a ROOT_URL no cluster can resolve, and Flux would report what looks like a network fault.
|
||||
# Pass it explicitly, e.g. for prod: HOST_NAT_IP=192.168.2.132 bash bootstrap-gitea.sh
|
||||
#
|
||||
# NOTE (DEV/libvirt ONLY, historical): firewalld's `libvirt` zone (owns virbrN) blocked
|
||||
# VM->host:3000 by default and had to be opened once with
|
||||
# sudo firewall-cmd --zone=libvirt --add-port=3000/tcp --permanent && sudo firewall-cmd --reload
|
||||
HOST_NAT_IP="${HOST_NAT_IP:-10.73.0.1}"
|
||||
# PROD needs NO firewall change: same-LAN, no libvirt zone involved (verified 2026-09-04).
|
||||
if [ -z "${HOST_NAT_IP:-}" ]; then
|
||||
echo "ERROR: HOST_NAT_IP is not set — it is this host's address AS SEEN FROM THE CLUSTER." >&2
|
||||
echo " prod: HOST_NAT_IP=192.168.2.132 (same LAN as the nodes, no NAT, no firewall change)" >&2
|
||||
echo " There is no safe default: the old one (10.73.0.1) is the RETIRED libvirt NAT." >&2
|
||||
exit 1
|
||||
fi
|
||||
ORG="${GITEA_ORG:-Home-Network}"
|
||||
REPO="${GITEA_REPO:-home-network-k3s-impl}"
|
||||
|
||||
@@ -75,8 +107,17 @@ cat <<EOF
|
||||
|
||||
Gitea bootstrap source ready (${DATA_DIR}).
|
||||
UI (host): ${BASE}/${ORG}/${REPO}
|
||||
Flux GitRepository (cluster): http://${HOST_NAT_IP}:${HTTP_PORT}/${ORG}/${REPO}.git (clusters/dev)
|
||||
Flux GitRepository (cluster): http://${HOST_NAT_IP}:${HTTP_PORT}/${ORG}/${REPO}.git
|
||||
(patch this url into the overlay being bootstrapped:
|
||||
clusters/prod today; clusters/dev has already cut over)
|
||||
Admin creds (host, 0600): ${ADMIN_CRED_FILE}
|
||||
Prerequisite: firewalld must allow VM->host:${HTTP_PORT} —
|
||||
Reachability: prod nodes reach ${HOST_NAT_IP}:${HTTP_PORT} directly on the LAN — no NAT, no
|
||||
firewall change (verified 2026-09-04). For libvirt-based clusters ONLY, firewalld's libvirt zone
|
||||
must allow VM->host:${HTTP_PORT} —
|
||||
ansible-playbook infra/ansible/playbooks/host-prereqs.yml --ask-become-pass
|
||||
REMINDER: the push above published the CURRENT HEAD as this repo's \`main\`. Whatever cluster points
|
||||
at this source will reconcile THAT revision.
|
||||
TEMPORARY BY DESIGN: this container is outside every cluster and is a single point of failure for
|
||||
any cluster reconciling from it. Cut over to in-cluster Gitea (docs/05); for prod the cutover
|
||||
procedure is written on the GitRepository patch in clusters/prod/kustomization.yaml.
|
||||
EOF
|
||||
+5
-5
@@ -24,9 +24,9 @@ env:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN # correct public clone/redirect URLs for the TLS-terminated host
|
||||
value: gitea.home.keithjackson.dev
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.home.keithjackson.dev/
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED # Phase 6 CI/CD: declared (not implicit upstream default) so the
|
||||
value: "true" # in-cluster act_runner (ns gitea-actions) registers against it
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL # resolve `uses: <org>/<action>@ref` from github.com (cluster
|
||||
@@ -104,7 +104,7 @@ lifecycle:
|
||||
# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker
|
||||
# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays
|
||||
# usable). The verify-then-retry loop is the real guarantee against "exit 0 but nothing stuck".
|
||||
GITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
|
||||
GITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration
|
||||
gitea_oidc_fail() {
|
||||
echo "[gitea-oidc] FAILED to register 'keycloak' login source: $1" >&2
|
||||
{ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true
|
||||
@@ -168,7 +168,7 @@ lifecycle:
|
||||
# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,
|
||||
# the moment its OIDC guard was fixed and the block actually ran for the first time.
|
||||
exit 0
|
||||
exposure: public # edge-forwarded git host at gitea.home.keithjackson.dev (docs/09; R-DNS/R-NET)
|
||||
exposure: public # edge-forwarded git host at gitea.${BASE_DOMAIN} (docs/09; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
- name: data
|
||||
@@ -178,7 +178,7 @@ storage:
|
||||
# the default /data/<name> would strand real state on the ephemeral rootfs (R-STATE-01)
|
||||
dependencies: [] # default-deny: reaches only DNS; git clients reach it north-south via the Gateway (R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: gitea.home.keithjackson.dev
|
||||
- host: gitea.${BASE_DOMAIN}
|
||||
port: 3000 # Gitea HTTP web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
|
||||
|
||||
@@ -17,7 +17,7 @@ name: jackett
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/jackett:latest
|
||||
exposure: internal # LAN-only web UI at jackett.home.keithjackson.dev (docs/13 row 4; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
||||
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
- name: config
|
||||
@@ -25,7 +25,7 @@ storage:
|
||||
class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: jackett.home.keithjackson.dev
|
||||
- host: jackett.${BASE_DOMAIN}
|
||||
port: 9117 # Jackett WebUI; protected at the Gateway by forward-auth (docs/09, docs/15)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
@@ -18,7 +18,7 @@ dependencies:
|
||||
port: 5432
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: keycloak.home.keithjackson.dev
|
||||
- host: keycloak.${BASE_DOMAIN}
|
||||
port: 8080 # HTTP listener; TLS terminates at the Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: native # it IS the IdP; own admin login is local + MFA (docs/15; ADR-0014). Documented exception.
|
||||
|
||||
@@ -50,13 +50,13 @@ env:
|
||||
# the vhost answer for the public FQDN; OVERWRITE* make Nextcloud emit https:// self-URLs even
|
||||
# though it speaks plain HTTP behind the proxy. ---
|
||||
- name: NEXTCLOUD_TRUSTED_DOMAINS
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEHOST
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEPROTOCOL
|
||||
value: https
|
||||
- name: OVERWRITECLIURL
|
||||
value: https://nextcloud.home.keithjackson.dev
|
||||
value: https://nextcloud.${BASE_DOMAIN}
|
||||
# --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential
|
||||
# `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret
|
||||
# (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it
|
||||
@@ -88,7 +88,7 @@ lifecycle:
|
||||
# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as
|
||||
# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.
|
||||
OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; }
|
||||
NEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
|
||||
NEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration
|
||||
nextcloud_oidc_fail() {
|
||||
echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2
|
||||
{ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true
|
||||
@@ -228,7 +228,7 @@ dependencies:
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
|
||||
ingressRoutes:
|
||||
- host: nextcloud.home.keithjackson.dev
|
||||
- host: nextcloud.${BASE_DOMAIN}
|
||||
port: 80 # apache HTTP; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ storage:
|
||||
class: longhorn # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03)
|
||||
dependencies: [] # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: pihole.home.keithjackson.dev
|
||||
- host: pihole.${BASE_DOMAIN}
|
||||
port: 80 # admin web UI, protected at the Gateway by forward-auth (docs/09)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ storage:
|
||||
# so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
|
||||
dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: plex.home.keithjackson.dev
|
||||
- host: plex.${BASE_DOMAIN}
|
||||
port: 32400 # Plex web UI + API served on 32400 (wildcard TLS at the Gateway; docs/08/09)
|
||||
auth:
|
||||
mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02)
|
||||
|
||||
@@ -32,7 +32,7 @@ name: qbittorrent
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at qbittorrent.home.keithjackson.dev (docs/13 row 13; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at qbittorrent.${BASE_DOMAIN} (docs/13 row 13; R-DNS/R-NET)
|
||||
egress: vpn # ALL public egress via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
- name: config
|
||||
@@ -40,7 +40,7 @@ storage:
|
||||
class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.home.keithjackson.dev
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
port: 8080 # qBittorrent WebUI; protected at the Gateway by forward-auth (docs/09, docs/15)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at radarr.home.keithjackson.dev (docs/13 row 3; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
@@ -29,7 +29,7 @@ dependencies:
|
||||
port: 9117
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: radarr.home.keithjackson.dev
|
||||
- host: radarr.${BASE_DOMAIN}
|
||||
port: 7878 # Radarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
|
||||
+9
-1
@@ -44,7 +44,15 @@ except ImportError: # pragma: no cover - explicit, actionable failure
|
||||
|
||||
# --- Label / naming conventions (match platform/kyverno/policies + platform/*/) --------
|
||||
LBL = "homelab.keithjackson.dev"
|
||||
ZONE = "home.keithjackson.dev"
|
||||
# The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal.
|
||||
# Every rendered manifest lands under a Flux path whose Kustomization declares
|
||||
# postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at
|
||||
# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev.
|
||||
# It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE —
|
||||
# the escape contract below reserves `$${...}` for expansions meant for a container's shell.
|
||||
# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
|
||||
# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
|
||||
ZONE = "${BASE_DOMAIN}"
|
||||
GATEWAY_NAME = "shared"
|
||||
GATEWAY_NS = "gateway"
|
||||
KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ dependencies:
|
||||
port: 32400
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: seerr.home.keithjackson.dev
|
||||
- host: seerr.${BASE_DOMAIN}
|
||||
port: 5055 # Overseerr web UI + API; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02)
|
||||
|
||||
@@ -58,7 +58,7 @@ env:
|
||||
- name: SEMAPHORE_ADMIN_NAME
|
||||
value: Semaphore Admin
|
||||
- name: SEMAPHORE_ADMIN_EMAIL
|
||||
value: admin@home.keithjackson.dev
|
||||
value: admin@${BASE_DOMAIN}
|
||||
# --- Persistent access-key encryption key (see header): FIXED base64 32-byte key from SOPS so the
|
||||
# SSH keys / credentials Semaphore stores in the DB stay decryptable across restarts. ---
|
||||
- name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
|
||||
@@ -110,7 +110,7 @@ dependencies:
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
|
||||
ingressRoutes:
|
||||
- host: semaphore.home.keithjackson.dev
|
||||
- host: semaphore.${BASE_DOMAIN}
|
||||
port: 3000 # Semaphore HTTP web/API; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ name: sonarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at sonarr.home.keithjackson.dev (docs/13 row 2; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
@@ -29,7 +29,7 @@ dependencies:
|
||||
port: 9117
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: sonarr.home.keithjackson.dev
|
||||
- host: sonarr.${BASE_DOMAIN}
|
||||
port: 8989 # Sonarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
|
||||
@@ -97,6 +97,27 @@ Kustomization: apps-wave-1 (./apps/wave-1, dependsOn apps-wave-0)
|
||||
Kustomization: apps-wave-N (./apps/wave-N, dependsOn apps-wave-N-1)
|
||||
```
|
||||
|
||||
### Second-order CRDs need a THIRD pass (`platform-config-cdi`, added 2026-09-05)
|
||||
|
||||
The controllers/configs split (`platform` → `platform-config`) assumes a CRD is registered by the
|
||||
controller **install**. CDI breaks that: `cdi-operator.yaml` registers only `cdis.cdi.kubevirt.io`,
|
||||
and cdi-operator creates `storageprofiles` / `datavolumes` / `datasources` **only after it has
|
||||
reconciled the `CDI` CR**. Since kustomize-controller server-side **dry-runs every object in a pass
|
||||
before applying any of them**, having the `CDI` CR and the CDI `StorageProfile`s in one pass was a
|
||||
**deadlock, not a race**: unknown kind → pass aborts → CR never applied → CRD never created →
|
||||
`platform-config` can never converge → every tier below it blocked. Invisible on dev (its CDI CRDs
|
||||
have existed for months); it cost 12h on prod's cold start.
|
||||
|
||||
So there is a third tier, `platform-config-cdi` (`./platform-config-cdi`, `dependsOn:
|
||||
platform-config`), and a rule:
|
||||
|
||||
> A CR belongs in `platform-config-cdi`, not `platform-config`, when the CRD it needs is created by a
|
||||
> **controller at runtime** rather than by a manifest in git.
|
||||
|
||||
`apps-vms` `dependsOn` it as well as `platform-config`, because CDI reads
|
||||
`StorageProfile/longhorn-vm` at DataVolume **provisioning** time — a VM disk provisioned before that
|
||||
profile exists is stamped RWO/Filesystem and is permanently not live-migratable.
|
||||
|
||||
Wave 0 `dependsOn` `platform-config` (transitively `platform`) so CNI/storage/TLS/DNS/gateway
|
||||
exist before any workload reconciles. The app tier is split into dependency-ordered WAVES
|
||||
(`catalog/render.py --emit-waves`): each `apps-wave-<N>` health-gates (wait:true) ONLY its own
|
||||
|
||||
@@ -4,8 +4,11 @@
|
||||
# NON-GATING (wait:false): unlike the app waves, this does NOT health-wait. A VM's first-boot image
|
||||
# import (CDI, minutes) + in-guest software install mean a VM can be intentionally long-not-Ready, so
|
||||
# health-gating it would wedge reconciliation for no benefit. VMs have no in-cluster dependents.
|
||||
# dependsOn platform-config so KubeVirt/CDI + the longhorn-vm StorageClass + CDI StorageProfiles
|
||||
# exist first. prune:true — this Kustomization solely owns the VM manifests (no adoption concern).
|
||||
# dependsOn platform-config so KubeVirt/CDI + the longhorn-vm StorageClass exist first, AND
|
||||
# platform-config-cdi so the CDI StorageProfiles do (2026-09-05: the StorageProfiles MOVED OUT of
|
||||
# platform-config into their own tier, so `dependsOn: platform-config` alone NO LONGER guarantees
|
||||
# them — see the dependsOn block). prune:true — this Kustomization solely owns the VM manifests (no
|
||||
# adoption concern).
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
@@ -19,6 +22,28 @@ spec:
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config
|
||||
# LOAD-BEARING, not decoration. apps/vms/unifi/20-datavolume.yaml provisions a DataVolume on
|
||||
# storageClassName `longhorn-vm`, and CDI reads StorageProfile/longhorn-vm AT PROVISIONING TIME to
|
||||
# decide the claim's accessModes/volumeMode. Without that profile's `claimPropertySets`
|
||||
# (ReadWriteMany + Block) the PVC is stamped RWO/Filesystem and the VM disk is PERMANENTLY not
|
||||
# live-migratable — a silent, unrecoverable-without-recreate loss. The profiles used to arrive
|
||||
# with platform-config; since 2026-09-05 they arrive with `platform-config-cdi`, so the edge has
|
||||
# to move with them. platform-config-cdi runs wait:true and StorageProfile has no status
|
||||
# conditions (kstatus -> Current on apply), so this edge cannot stall this tier.
|
||||
#
|
||||
# HONEST LIMIT: CDI provisions from the profile's STATUS, which its storageprofile-controller
|
||||
# derives from our spec. This edge proves the spec is applied, not that CDI has folded it into
|
||||
# status — that is a watch-triggered round-trip of seconds against a ~30s dependency requeue, so
|
||||
# it is a timing argument rather than a guarantee. See clusters/base/platform-config-cdi.yaml.
|
||||
#
|
||||
# This edge also makes the StorageProfile prune/recreate transition safe: platform-config's prune
|
||||
# deletes the two profiles for the ~30-60s before platform-config-cdi re-applies them, and this
|
||||
# dependsOn is what guarantees Flux cannot create a DataVolume inside that window.
|
||||
#
|
||||
# The platform-config edge above is kept although platform-config-cdi already dependsOn it: the
|
||||
# edges are a set, there is no cycle, and stating both means a future edit to platform-config-cdi
|
||||
# cannot silently drop this tier's KubeVirt/StorageClass gate.
|
||||
- name: platform-config-cdi
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
|
||||
@@ -17,9 +17,63 @@ data:
|
||||
CLUSTER_ENV: "base" # dev | prod (overlays set this)
|
||||
|
||||
# --- DNS / TLS ---
|
||||
BASE_DOMAIN: "home.keithjackson.dev" # platform hostname suffix (same both envs; split-horizon)
|
||||
DNS_ZONE: "keithjackson.dev" # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it);
|
||||
# ExternalDNS domain-filter must be the zone, not the subdomain
|
||||
# THE platform hostname suffix. Every public hostname the cluster serves is `<svc>.${BASE_DOMAIN}`,
|
||||
# the wildcard cert is `*.${BASE_DOMAIN}`, coredns-custom answers the `${BASE_DOMAIN}` zone with
|
||||
# ${GATEWAY_VIP}, and every OIDC issuer/redirect URI is built from it.
|
||||
#
|
||||
# THIS KNOB WAS DEAD UNTIL 2026-09-04 — declared here but referenced by NO manifest, because every
|
||||
# hostname was hardcoded. That is what made the 2026-09-04 incident possible: prod and dev were
|
||||
# authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns
|
||||
# (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM.
|
||||
# See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN
|
||||
# subdomain, so the two clusters can no longer describe the same name.
|
||||
#
|
||||
# dev -> home.keithjackson.dev (UNCHANGED — see below)
|
||||
# prod -> prod.home.keithjackson.dev
|
||||
#
|
||||
# DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on
|
||||
# `dev.home.keithjackson.dev` with apex CNAMEs `<svc>.home` -> `<svc>.prod.home`, but moving dev
|
||||
# relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and
|
||||
# the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2
|
||||
# (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as
|
||||
# GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates
|
||||
# it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre-
|
||||
# tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths).
|
||||
BASE_DOMAIN: "home.keithjackson.dev"
|
||||
|
||||
# ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind
|
||||
# EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered,
|
||||
# so it cannot be created, updated or deleted regardless of TXT ownership.
|
||||
#
|
||||
# DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev`
|
||||
# apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter
|
||||
# that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching
|
||||
# record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged
|
||||
# from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically.
|
||||
#
|
||||
# PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE
|
||||
# (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned
|
||||
# chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED
|
||||
# — it can never delete another environment's records — and the fallback is to set prod's value
|
||||
# back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard.
|
||||
EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev"
|
||||
|
||||
# ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT:
|
||||
# neither cluster set it, so both inherited the chart default `default`, and ExternalDNS treats a
|
||||
# record whose TXT registry entry carries ITS OWN owner id as its own to reconcile — and with
|
||||
# `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on
|
||||
# one zone therefore garbage-collect each other. Each environment MUST have a distinct value.
|
||||
#
|
||||
# DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare
|
||||
# under owner `default`; changing dev's value would make dev's own external-dns stop recognising
|
||||
# them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it.
|
||||
#
|
||||
# THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that
|
||||
# forgets this knob must not silently inherit dev's identity — that is precisely the incident. With
|
||||
# this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a
|
||||
# visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS.
|
||||
EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars"
|
||||
|
||||
ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
|
||||
GATEWAY_VIP: "192.168.1.53" # in-cluster split-horizon target (R-DNS-01/03): the L2-announced
|
||||
# ingress VIP of the `shared` Gateway. MUST equal the FIRST address of
|
||||
@@ -47,10 +101,37 @@ data:
|
||||
# ship the literal string into the cluster.
|
||||
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml).
|
||||
# [parity] MUST equal the Ansible bootstrap's k3s_api_host (infra/ansible/roles/cilium/defaults/
|
||||
# main.yml -> cilium-values.yaml.j2, already inventory-parameterized). Wrong value = Cilium cannot
|
||||
# reach the API server on adopt and the CNI breaks. Port stays 6443 in both envs, so it is NOT a knob.
|
||||
K3S_API_HOST: "192.168.1.50"
|
||||
#
|
||||
# *** THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS. *** Operator decision
|
||||
# 2026-09-06. It was each environment's PRIMARY NODE IP until then, which made the CNI's
|
||||
# control-plane path depend on one machine in a cluster whose entire purpose is HA: verified on
|
||||
# live prod, all three cilium-agents carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1's
|
||||
# own address) while kube-vip held 192.168.2.9. Losing that node leaves the datapath running off
|
||||
# Cilium's cache while NO service or policy update lands — a silent, unbounded staleness, not an
|
||||
# outage you would notice. A virtual address that follows the control plane is the fix, and it is
|
||||
# what kube-vip was stood up to provide.
|
||||
#
|
||||
# dev -> 192.168.1.49 (= kube_vip_address, group_vars/all.yml; base default, below)
|
||||
# prod -> 192.168.2.9 (= kube_vip_address, group_vars/k3s_prod/vars.yml)
|
||||
#
|
||||
# THE NAME IS NOW SLIGHTLY WRONG AND IS KEPT ANYWAY: renaming the knob would be a second,
|
||||
# unrelated change to every substituting Kustomization at the same time as a live CNI endpoint
|
||||
# move. Read it as "the API host Cilium dials", which is what it always meant.
|
||||
#
|
||||
# [parity] MUST equal the Ansible bootstrap's **k3s_apiserver_endpoint** for the SAME environment
|
||||
# (infra/ansible/inventory/group_vars/all.yml -> roles/cilium/templates/cilium-values.yaml.j2,
|
||||
# already inventory-parameterized). NOT k3s_api_host — that variable still exists and still holds
|
||||
# the FOUNDER's address, because the cold-bootstrap `--server` join needs a machine that is
|
||||
# already up before any VIP exists. The two are deliberately separate; see the TWO API ADDRESSES
|
||||
# block at the top of group_vars/all.yml.
|
||||
#
|
||||
# Wrong value = Cilium cannot reach the API server on adopt and the CNI breaks. Port stays 6443
|
||||
# in both envs, so it is NOT a knob.
|
||||
#
|
||||
# TLS needs nothing: k3s_tls_sans has always carried kube_vip_address alongside every node IP, so
|
||||
# both addresses are already in the serving cert (verified live 2026-09-06 — prod answers through
|
||||
# https://192.168.2.9:6443 under full CA verification). No cert reissue, no k3s restart.
|
||||
K3S_API_HOST: "192.168.1.49"
|
||||
|
||||
# CiliumLoadBalancerIPPool `default-pool` (platform-config/cilium/l2-ip-pool.yaml): the VIP range
|
||||
# L2-announced on the flat LAN. Must sit OUTSIDE the node IPs and any DHCP scope, and must match the
|
||||
|
||||
@@ -14,6 +14,15 @@ resources:
|
||||
- cilium.yaml # Flux Kustomization -> ./platform/cilium (dependsOn flux-system, wait:true)
|
||||
- platform.yaml # Flux Kustomization -> ./platform (CONTROLLERS: HelmReleases / CRDs)
|
||||
- platform-config.yaml # Flux Kustomization -> ./platform-config (CONFIGS: CRs; dependsOn platform)
|
||||
# SECOND-ORDER CDI configs — CRs whose CRD is created by a controller AT RUNTIME, not by any
|
||||
# manifest in git. cdi-operator.yaml registers only `cdis.cdi.kubevirt.io`; `storageprofiles` (and
|
||||
# datavolumes, datasources, …) appear only after cdi-operator reconciles the `CDI` CR that
|
||||
# platform-config applies. Because Flux dry-runs every object in a pass before applying any of them,
|
||||
# keeping the StorageProfiles in platform-config was a TRUE DEADLOCK (unknown kind -> pass aborts ->
|
||||
# CR never applied -> CRD never created -> tier never converges), invisible on dev and fatal on any
|
||||
# cold cluster — observed on prod 2026-09-05. This tier is the durable fix; `standup.sh crds` stays
|
||||
# as the belt-and-braces repair for clusters already wedged. See clusters/base/platform-config-cdi.yaml.
|
||||
- platform-config-cdi.yaml # Flux Kustomization -> ./platform-config-cdi (dependsOn platform-config)
|
||||
# App tier, split into dependency-ordered WAVES (was the single ./apps `apps` Kustomization).
|
||||
# Each apps-wave-<N> dependsOn wave N-1 (wave 0 dependsOn platform-config) and health-gates
|
||||
# (wait:true) ONLY its own wave, so a crashloop in a later wave can't stall an earlier one.
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
---
|
||||
# Flux Kustomization: SECOND-ORDER CDI CONFIGS — reconciles ./platform-config-cdi.
|
||||
# Serves R-GITOPS-01 (all desired state in git, reconciled by Flux) and R-ENV-02.
|
||||
#
|
||||
# WHY THIS TIER EXISTS — the durable fix for the 2026-09-05 prod CDI DEADLOCK
|
||||
#
|
||||
# `platform-config` applied the `CDI` CR and the CDI `StorageProfile`s in the SAME pass.
|
||||
# kustomize-controller server-side DRY-RUNS EVERY object before applying ANY of them, and
|
||||
# `storageprofiles.cdi.kubevirt.io` does not exist until cdi-operator has reconciled that `CDI` CR.
|
||||
# So the unknown kind aborted the whole tier → the CR was never applied → the CRD was never created →
|
||||
# platform-config could NEVER converge and every tier below it stayed blocked. Not a race: a
|
||||
# deadlock. cdi-operator was Available for 12h on prod with no CR. Invisible on dev (its CDI CRDs
|
||||
# have existed for months), FATAL on any cold cluster. See DECISIONS-LOG 2026-09-05.
|
||||
#
|
||||
# `bootstrap/standup.sh crds` pre-applies the `CDI` CR to break it by hand. That is a WORKAROUND and
|
||||
# stays as belt-and-braces for clusters already wedged; THIS Kustomization is the fix that makes the
|
||||
# graph order itself, so a cold cluster needs no human in the loop.
|
||||
#
|
||||
# THE ORDERING GUARANTEE, precisely: `dependsOn` is REVISION-gated, not merely Ready-gated (verified
|
||||
# against kustomize-controller v1.4.0 `kustomization_controller.go :: checkDependencies`, our
|
||||
# version): sharing a GitRepository with the dependency, this Kustomization cannot reconcile revision
|
||||
# N until `platform-config` has APPLIED revision N — and, because platform-config runs `wait: true`,
|
||||
# not until the `CDI` CR it applied is HEALTHY. cdi-operator creates the remaining CDI CRDs as part of
|
||||
# reaching that state, so by the time this tier's first dry-run runs, `StorageProfile` is a known
|
||||
# kind. One edge, no pre-apply.
|
||||
#
|
||||
# WHY NOT just move the CDI CR earlier, into `platform`? Because `platform` is the tier the CNI gates
|
||||
# and everything else waits on: putting CDI's health on that critical path is the mistake the
|
||||
# oauth2-proxy/headlamp splits were made to undo. The CR is correctly a platform-config CR; it is the
|
||||
# StorageProfiles that were in the wrong pass.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: platform-config-cdi
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
# 2m (short, deliberately). kustomize-controller caches API discovery, so on a COLD cluster the
|
||||
# very first attempt here can still fail `no matches for kind "StorageProfile"` even though the CRD
|
||||
# now exists — a stale RESTMapper, not a missing CRD (the failure mode `standup.sh crds` warns
|
||||
# about). The controller resets the mapper and the retry succeeds, so the only cost of that stale
|
||||
# cache is one retryInterval. Keep it short.
|
||||
retryInterval: 2m
|
||||
# 2m is ample: this tier applies two tiny cluster-scoped CRs. There is no workload to roll out, no
|
||||
# image to pull and no controller to become Available, so a slow apply here means a real error, not
|
||||
# a slow rollout — failing fast surfaces it instead of burning 5m per attempt.
|
||||
timeout: 2m
|
||||
# TRUE, and this tier is the SOLE declarer of both StorageProfiles (checked repo-wide), so prune is
|
||||
# component-scoped: the stale set can only ever be a StorageProfile this file stopped rendering.
|
||||
# It is NOT the platform/cilium case — nothing here declares a shared Namespace, or any object a
|
||||
# second path also declares. See the transition note at the bottom of this file.
|
||||
prune: true
|
||||
# TRUE, but understand exactly what it buys and what it does NOT.
|
||||
#
|
||||
# CANNOT STALL: `StorageProfile`'s status type (CDI v1beta1 StorageProfileStatus: storageClass,
|
||||
# provisioner, claimPropertySets, cloneStrategy, dataImportCronSourceFormat) carries NO
|
||||
# `conditions` and NO `observedGeneration`, so kstatus's generic path reports it Current the moment
|
||||
# it is applied. `wait: true` therefore adds a Ready condition without adding a way to hang — which
|
||||
# is why it is safe to let `apps-vms` gate on it (contrast the headlamp gating stall).
|
||||
# If a resource that CAN be unhealthy is ever added to ./platform-config-cdi, re-derive this:
|
||||
# `apps-vms` waits on it.
|
||||
#
|
||||
# WHAT IT DOES NOT PROVE (honest gap): CDI reads the profile's **status**, not its spec, when it
|
||||
# provisions a claim (status.claimPropertySets / status.cloneStrategy), and CDI's
|
||||
# storageprofile-controller computes that status from our spec override plus inference. So Ready
|
||||
# here means "the spec override is accepted by the API server", not "CDI has folded it into
|
||||
# status". That reconcile is a watch-triggered round-trip of seconds, and the dependency requeue
|
||||
# below it is ~30s, so a DataVolume cannot realistically beat it — but it is a timing argument, not
|
||||
# a guarantee, and no `wait` setting can turn it into one (there is no condition to wait on).
|
||||
wait: true
|
||||
dependsOn:
|
||||
# platform-config applies the `CDI` CR and (wait:true) waits for it to be healthy — which is what
|
||||
# causes cdi-operator to create `storageprofiles.cdi.kubevirt.io`. Revision-gated, so this is an
|
||||
# ordering guarantee and not a hope. Transitively this also chains platform -> cilium ->
|
||||
# flux-system, so `cluster-vars` at revision N is applied before this renders revision N.
|
||||
- name: platform-config
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform-config-cdi
|
||||
# NO SOPS `decryption` stanza: ../platform/kubevirt/cdi-storageprofiles.yaml is two plain CRs with
|
||||
# no Secret and no *.sops.yaml, so neither overlay needs to patch one in (unlike
|
||||
# oauth2-proxy/headlamp/crafty/velero, which each carry an encrypted Secret). Nothing to add to
|
||||
# clusters/dev or clusters/prod for this tier.
|
||||
#
|
||||
# substituteFrom cluster-vars: the manifest currently contains NO `${TOKEN}`, so this stanza
|
||||
# substitutes nothing today. It is declared anyway for two concrete reasons, not for symmetry:
|
||||
# 1. COVERAGE. conformance/tests/l0-static/flux-postbuild-substitution/check.sh only audits paths
|
||||
# whose Kustomization declares substituteFrom. Omitting it would move
|
||||
# cdi-storageprofiles.yaml OUT of the audited set — a blind spot where a future `${TOKEN}`
|
||||
# could ship as a literal string, precisely the class of bug that check exists to stop.
|
||||
# 2. FAIL-CLOSED. kustomize-controller runs --feature-gates=StrictPostBuildSubstitutions=true, so
|
||||
# with this stanza an undefined token FAILS this Kustomization loudly instead of rendering
|
||||
# empty. Without the stanza there is no substitution at all and the literal ships silently.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# ── FIRST RECONCILE AFTER MERGE: a TRANSIENT DELETE-THEN-RECREATE. READ BEFORE MERGING. ──────────
|
||||
# Moving cdi-storageprofiles.yaml out of ./platform-config drops both StorageProfiles from THAT
|
||||
# Kustomization's inventory, and platform-config has `prune: true` — so its next apply DELETES
|
||||
# StorageProfile/longhorn and StorageProfile/longhorn-vm, and this Kustomization recreates them on
|
||||
# the following reconcile (revision-gated: strictly after platform-config's apply, so there is no
|
||||
# dual-ownership fight — the objects are pruned before they are re-applied under this tier's
|
||||
# `kustomize.toolkit.fluxcd.io/name=platform-config-cdi` label).
|
||||
#
|
||||
# Why that is acceptable here — and it is NOT the platform/cilium case:
|
||||
# * Neither object is the sole (or any) declarer of a shared object. No Namespace, no ConfigMap,
|
||||
# nothing else in the repo declares a StorageProfile. The stale set is exactly two
|
||||
# cluster-scoped CDI CRs.
|
||||
# * The objects do not actually go missing: CDI's storageprofile-controller creates a
|
||||
# StorageProfile per StorageClass and recreates a deleted one, with an EMPTY `.spec` and
|
||||
# inferred `.status`. So the transient is "our .spec overrides are absent", not "the object is
|
||||
# absent".
|
||||
# * A StorageProfile is read at DataVolume/PVC PROVISIONING and CLONE time only. Existing PVCs,
|
||||
# existing DataVolumes and running VMs do not consult it again, so nothing already provisioned
|
||||
# is disturbed and no data path is touched.
|
||||
# * The only exposure is a DataVolume/clone STARTED inside the window: it would clone by `copy`
|
||||
# instead of `snapshot`, and on `longhorn-vm` be stamped RWO/Filesystem instead of RWX/Block —
|
||||
# permanently non-live-migratable for that disk. Flux cannot cause that, because `apps-vms` now
|
||||
# dependsOn THIS tier (clusters/base/apps-vms.yaml), so no Flux-managed DataVolume can be
|
||||
# created while the overrides are absent. A human running `virtctl`/`kubectl` in that minute
|
||||
# could.
|
||||
#
|
||||
# PRECONDITION THE OPERATOR MUST CHECK, because it is what bounds the window: the recreate cannot
|
||||
# happen until `platform-config` is Ready, and platform-config runs `wait: true` over the whole
|
||||
# config tier (ClusterIssuers, wildcard Certificate, Gateway, Keycloak, CNPG, KubeVirt, CDI…). On a
|
||||
# healthy cluster that is seconds and the window is ~30-60s (kustomize-controller's
|
||||
# --requeue-dependency default is 30s). If platform-config is NOT Ready when this merges, the
|
||||
# StorageProfiles are pruned and stay at CDI defaults until it recovers. So:
|
||||
# `flux get kustomization platform-config` must be Ready BEFORE merging.
|
||||
# If it is not, do the two-step the cilium move used instead: merge with `platform-config.prune`
|
||||
# temporarily set to `false` (the objects are then dropped from its inventory but NOT deleted, and
|
||||
# this tier ADOPTS them with zero window), verify the ownership label transferred, then restore
|
||||
# `prune: true` in a second commit.
|
||||
@@ -148,6 +148,33 @@ patches:
|
||||
- op: replace
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# --- DNS identity (R-DNS-01/02/03) -----------------------------------------------------
|
||||
# DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly
|
||||
# like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running.
|
||||
# Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its
|
||||
# Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev
|
||||
# (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to
|
||||
# prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did
|
||||
# not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration,
|
||||
# not a variable change: infra/PROD-PROVISIONING.md §10.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "home.keithjackson.dev"
|
||||
# ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself,
|
||||
# and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped
|
||||
# the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
value: "keithjackson.dev"
|
||||
# DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT
|
||||
# registry under owner `default` (the external-dns chart default, which is what both clusters
|
||||
# silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records
|
||||
# BY this string: any other value and it stops recognising every record it manages, then
|
||||
# re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the
|
||||
# whole point. The base default is a non-colliding sentinel, so this line is load-bearing.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "default"
|
||||
# --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) -----------------------------
|
||||
# EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it
|
||||
# equals the base default. These values are what the LIVE dev cluster is running right now, so
|
||||
@@ -160,11 +187,25 @@ patches:
|
||||
- op: replace
|
||||
path: /data/GATEWAY_VIP
|
||||
value: "192.168.1.53"
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — the primary control-plane node
|
||||
# (k3s-1). [parity] equals the Ansible inventory's k3s_api_host for dev.
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials — DEV'S CONTROL-PLANE VIP,
|
||||
# the address kube-vip floats, NOT the primary node .50.
|
||||
# [parity] equals the Ansible inventory's k3s_apiserver_endpoint for dev
|
||||
# (= kube_vip_address, group_vars/all.yml). See clusters/base/cluster-vars.yaml for why.
|
||||
#
|
||||
# *** THE ONE EXCEPTION TO "NEVER LET THIS BLOCK DRIFT FROM THE LIVE CLUSTER" ***
|
||||
# Every other value in this block is what dev is running right now. This one is not:
|
||||
# the live dev cilium-agents dial 192.168.1.50 (k3s-1). Changing it here is a
|
||||
# DELIBERATE, OPERATOR-APPROVED move (2026-09-06) and reconciling it ROLLS THE CILIUM
|
||||
# AGENT DAEMONSET on dev — one node at a time (updateStrategy.maxUnavailable: 1, kept
|
||||
# in parity for exactly this reason). Do not push this to dev-live casually, and not
|
||||
# while dev's DNS migration is mid-flight; the L2 datapath survives an agent restart
|
||||
# but a wrong value here is the 2026-09-01 incident. .49 is in dev's serving-cert SANs
|
||||
# already (k3s_tls_sans) and dev's own kubeconfig validates against it via
|
||||
# tls-server-name, so TLS is not the risk — the risk is reachability, and .49 is on the
|
||||
# same LAN as the nodes and ARP-announced by kube-vip on every server.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.1.50"
|
||||
value: "192.168.1.49"
|
||||
# Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs
|
||||
# (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free.
|
||||
- op: replace
|
||||
|
||||
@@ -115,8 +115,20 @@ patches:
|
||||
name: sops-age
|
||||
# --- GitOps source (operator decision 2026-09-01: prod tracks the SAME BRANCH as dev) ---
|
||||
#
|
||||
# *** BOOTSTRAP BLOCKER — this URL CANNOT work on a cold prod cluster. READ BEFORE BRING-UP. ***
|
||||
# ############################################################################################
|
||||
# ### ###
|
||||
# ### *** T E M P O R A R Y B O O T S T R A P S O U R C E — NOT PERMANENT *** ###
|
||||
# ### ###
|
||||
# ### The url below is a Gitea container running ON THE OPERATOR'S WORKSTATION, outside ###
|
||||
# ### any cluster. It exists ONLY to get prod's first reconcile off the ground. It MUST ###
|
||||
# ### be cut over to the in-cluster URL once Gitea runs on prod. If you are reading this ###
|
||||
# ### and prod has been serving traffic for months, THE CUTOVER WAS NEVER DONE — that is ###
|
||||
# ### a live single point of failure, not a settled design. See "CUTOVER" below. ###
|
||||
# ### ###
|
||||
# ############################################################################################
|
||||
#
|
||||
# WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the
|
||||
# unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`).
|
||||
# Two independent reasons, both consequences of the split-horizon design:
|
||||
# 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's
|
||||
# GitOps source lives inside the very cluster slated for decommission. Gitea must move to
|
||||
@@ -126,13 +138,46 @@ patches:
|
||||
# 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve
|
||||
# gitea.home.keithjackson.dev to itself and never reach a repository.
|
||||
#
|
||||
# Bring-up therefore needs a TEMPORARY source (the bootstrap host-Gitea, or an off-cluster
|
||||
# mirror), with a cutover to the in-cluster URL once Gitea actually runs on prod — exactly the
|
||||
# sequence docs/05 describes for dev's own Phase-6 cutover. Tracked in PROD-PROVISIONING.md §7.
|
||||
# WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction):
|
||||
# bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the
|
||||
# operator's workstation, Gitea 1.22.6, listening on 192.168.2.132:3000. All three prod nodes
|
||||
# reach it DIRECTLY on the LAN: no libvirt NAT hop (that network is retired) and no firewall
|
||||
# change was needed. This is the same "local bootstrap source" pattern dev itself used before
|
||||
# its Phase-6 cutover (docs/05 "Source-of-truth").
|
||||
#
|
||||
# PLAIN HTTP, AND WHY THAT IS ACCEPTED HERE: the bootstrap Gitea has no TLS, so there is no cert
|
||||
# for Flux to trust and no caCertSecretRef to wire. Traffic is LAN-local between prod nodes and
|
||||
# the workstation, the repo carries NO plaintext secrets (everything sensitive is SOPS-encrypted,
|
||||
# .sops.yaml), and the window is the bring-up only. Do NOT extend this posture past cutover.
|
||||
#
|
||||
# NO secretRef, DELIBERATELY: the repo is PUBLIC on the bootstrap Gitea, so anonymous clone
|
||||
# works and Flux needs no git credentials. VERIFIED 2026-09-04 by unauthenticated request:
|
||||
# curl http://192.168.2.132:3000/api/v1/repos/Home-Network/home-network-k3s-impl -> "private":false
|
||||
# git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git -> refs/heads/main
|
||||
# If the GitRepository ever reports an auth failure that looks like a network problem, RE-CHECK
|
||||
# THAT FIRST: a private repo here fails exactly that way.
|
||||
#
|
||||
# *** CUTOVER — the exit condition, mirroring dev's Phase-6 cutover (docs/05) ***
|
||||
# WHEN: Gitea runs on prod (reconciled in-cluster) and answers gitea.home.keithjackson.dev
|
||||
# through prod's own Gateway VIP with a publicly-trusted cert.
|
||||
# DO: 1. push/mirror this repo into the prod in-cluster Gitea;
|
||||
# 2. change the url below to
|
||||
# https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
# and DELETE this whole TEMPORARY block (dev's equivalent stanza, ~15 lines above in
|
||||
# clusters/dev/kustomization.yaml, is what "done" looks like);
|
||||
# 3. confirm `flux get sources git -n flux-system` reconciles the new revision;
|
||||
# 4. only THEN stop the workstation container. Keep it stopped-not-deleted as recovery,
|
||||
# exactly as dev retained its bootstrap Gitea.
|
||||
# UNTIL THEN: prod's reconciliation depends on a container on a workstation that may be
|
||||
# powered off. Flux keeps the last-reconciled state running, but NO new revision lands while
|
||||
# it is down. Tracked as an open blocker in infra/PROD-PROVISIONING.md §7.
|
||||
#
|
||||
# The BRANCH below is settled and correct: prod tracks `main`, the same branch as dev.
|
||||
# CONSEQUENCE, recorded deliberately: once prod is live, EVERY MERGE TO main RECONCILES BOTH
|
||||
# CLUSTERS. There is no promotion step and no soak window — dev and prod move together.
|
||||
# NOTE for bring-up: bootstrap-gitea.sh pushes the operator's CURRENT HEAD to the bootstrap
|
||||
# Gitea's `main`, whatever branch that is. Before bootstrapping prod, confirm that remote `main`
|
||||
# is the revision you intend prod to run (`git ls-remote` above, against local `main`).
|
||||
- patch: |-
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: GitRepository
|
||||
@@ -140,7 +185,9 @@ patches:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
# TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
|
||||
# above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
|
||||
url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
- target:
|
||||
@@ -154,6 +201,63 @@ patches:
|
||||
- op: replace
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# ==========================================================================================
|
||||
# PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident.
|
||||
# ==========================================================================================
|
||||
# PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to
|
||||
# home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME
|
||||
# public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one
|
||||
# of which already happened:
|
||||
# 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit
|
||||
# default dev had) concluded it owned dev's records and DELETED EVERY ONE of them.
|
||||
# 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev`
|
||||
# wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would
|
||||
# have raced each other's `_acme-challenge` TXT records on that one zone.
|
||||
# Moving prod to its own suffix dissolves both: the two clusters can no longer name the same
|
||||
# record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here.
|
||||
#
|
||||
# dev -> home.keithjackson.dev (UNCHANGED — dev is live; phase 2 moves it)
|
||||
# prod -> prod.home.keithjackson.dev (this knob)
|
||||
#
|
||||
# CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is
|
||||
# gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so
|
||||
# on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make
|
||||
# `<svc>.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they
|
||||
# would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
|
||||
# The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
|
||||
# set Ansible-side from that variable and cannot read this one. If they disagree, kubectl SSO
|
||||
# trusts a different Keycloak than the one prod actually runs.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "prod.home.keithjackson.dev"
|
||||
# SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's
|
||||
# OWN subtree, so prod cannot create, update or delete any record outside it no matter what the
|
||||
# TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for
|
||||
# ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped
|
||||
# the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no
|
||||
# matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's
|
||||
# records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on
|
||||
# EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
# MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG,
|
||||
# 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the
|
||||
# record then skipped it with "no hosted zone matching record DNS Name", so prod would
|
||||
# publish NOTHING. Isolation from dev does NOT come from this filter — it comes from
|
||||
# (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and
|
||||
# (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records.
|
||||
# That pairing is what the 2026-09-04 incident needed and this filter cannot provide.
|
||||
value: "keithjackson.dev"
|
||||
# SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so
|
||||
# both inherited the chart default `default`; prod then treated dev's TXT-registry entries as
|
||||
# its own and, under policy: sync, deleted the records behind them. This value MUST differ from
|
||||
# dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and
|
||||
# cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "prod"
|
||||
- op: replace
|
||||
path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
|
||||
value: "3"
|
||||
@@ -236,14 +340,32 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CILIUM_LB_POOL_STOP
|
||||
value: "192.168.2.30"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
# k3s API endpoint Cilium's kube-proxy replacement dials. This is the prod CONTROL-PLANE VIP
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10. It is owned
|
||||
# Ansible-side (prod inventory k3s_api_host) and MUST equal that value — [parity] contract in
|
||||
# platform/cilium/helmrelease.yaml. Not yet decided.
|
||||
# (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10.
|
||||
#
|
||||
# *** DECIDED 2026-09-06 by the operator: the VIP, as intended. *** (This block previously
|
||||
# read "Not yet decided" while shipping .10 — the founder's own address — and that is what
|
||||
# the LIVE prod cluster is running: all three cilium-agents carry
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip holds .9, verified 2026-09-06. On a
|
||||
# 3-node HA cluster that makes the CNI's control-plane path depend on k3s-prod-1: lose it and
|
||||
# the datapath keeps running off Cilium's cache while no service or policy update lands, with
|
||||
# nothing looking broken. The VIP is a virtual address that reaches the cluster wherever the
|
||||
# control plane is, which is the whole reason kube-vip exists here.)
|
||||
#
|
||||
# It is owned Ansible-side by **k3s_apiserver_endpoint** (group_vars/all.yml, derived from
|
||||
# this environment's kube_vip_address = 192.168.2.9 in group_vars/k3s_prod/vars.yml) and MUST
|
||||
# equal that value — [parity] contract in platform/cilium/helmrelease.yaml. NOT k3s_api_host:
|
||||
# that variable still exists and still holds the FOUNDER's .10, because the cold-bootstrap
|
||||
# `--server` join needs a host that is up before any VIP is floated. Two addresses, two
|
||||
# variables, deliberately.
|
||||
#
|
||||
# .9 sits OUTSIDE the Cilium LB pool (.20-.30) and is not a node address, so kube-vip and
|
||||
# LB-IPAM can never be handed the same IP — asserted by
|
||||
# conformance/tests/l0-static/ha-control-plane-as-code.
|
||||
- op: replace
|
||||
path: /data/K3S_API_HOST
|
||||
value: "192.168.2.10"
|
||||
value: "192.168.2.9"
|
||||
# --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------
|
||||
# Pinned L4 VIPs, both carved out of the pool by lbipam.cilium.io/ips. Not yet assigned for
|
||||
# prod. Leaving the dev literals in place would hand a prod LoadBalancer a dev-LAN address, so
|
||||
# they stay placeholders; whatever is chosen MUST fall inside [POOL_START, POOL_STOP] and not
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8080
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -76,8 +76,8 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://paas-demo.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
+3
-3
@@ -91,7 +91,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -99,8 +99,8 @@ spec:
|
||||
- --allowed-group=media-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
@@ -91,7 +91,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -99,8 +99,8 @@ spec:
|
||||
- --allowed-group=media-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
+209
-9
@@ -1172,6 +1172,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
| `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target |
|
||||
| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. |
|
||||
|
||||
- [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both*
|
||||
overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the
|
||||
@@ -1193,12 +1194,15 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
as dev**. Eight `decryption: {provider: sops, secretRef: sops-age}` patches mirror dev
|
||||
exactly. The `sops-age` Secret is still created OUT OF BAND on the prod cluster
|
||||
(`bootstrap/sops/install-age-key.sh`) — the overlay references it by name only.
|
||||
- [~] **GitRepository source** — BRANCH settled, URL still blocked. Operator decision
|
||||
2026-09-01: **prod tracks `main`, the same branch as dev**, because dev is temporary and
|
||||
will be decommissioned once prod is stable. Recorded consequence: once prod is live,
|
||||
**every merge to `main` reconciles BOTH clusters** — no promotion step, no soak window.
|
||||
The URL is patched in but **cannot work on a cold prod cluster**, for two independent
|
||||
reasons found 2026-09-01:
|
||||
- [~] **GitRepository source** — **MITIGATED 2026-09-04, NOT SOLVED.** Branch settled; the
|
||||
in-cluster URL is still unusable, so prod now points at a **temporary bootstrap source**.
|
||||
Operator decision 2026-09-01: **prod tracks `main`, the same branch as dev**, because dev
|
||||
is temporary and will be decommissioned once prod is stable. Recorded consequence: once
|
||||
prod is live, **every merge to `main` reconciles BOTH clusters** — no promotion step, no
|
||||
soak window.
|
||||
|
||||
**Why the in-cluster URL (`https://gitea.home.keithjackson.dev/...`) cannot work on a
|
||||
cold prod cluster** — two independent reasons, found 2026-09-01, both still true:
|
||||
1. **Self-hosting circularity** — Gitea runs on the DEV cluster, the one slated for
|
||||
decommission. Gitea must move to prod (or off-cluster) BEFORE dev is retired, or prod
|
||||
loses its source of truth. **This is the decommission plan's first task, not its last.**
|
||||
@@ -1206,9 +1210,56 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
`home.keithjackson.dev` zone with `${GATEWAY_VIP}`, which on prod is `192.168.2.20`:
|
||||
prod's OWN gateway, where no Gitea listens. Prod's Flux would resolve
|
||||
`gitea.home.keithjackson.dev` to itself and never reach a repository.
|
||||
Bring-up needs a TEMPORARY source (bootstrap host-Gitea, or an off-cluster mirror) and a
|
||||
cutover once Gitea runs on prod — the same sequence `docs/05` describes for dev's own
|
||||
Phase-6 cutover.
|
||||
|
||||
**The mitigation now in `clusters/prod/kustomization.yaml`** (operator directed this to be
|
||||
stood up, 2026-09-04):
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| GitRepository url | `http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` |
|
||||
| Branch | `main` (unchanged — same branch as dev) |
|
||||
| What it is | Gitea 1.22.6 in a **podman container on the operator's workstation**, started by `bootstrap/gitea/bootstrap-gitea.sh` with `HOST_NAT_IP=192.168.2.132` |
|
||||
| Reachability | All three prod nodes reach it **directly on the LAN** — no NAT hop (the libvirt network is retired), **no firewall change needed**. Verified from all three nodes. |
|
||||
| Auth | **None, deliberately.** The repo is PUBLIC on that Gitea, so Flux needs no `secretRef`. Verified anonymously: `curl .../api/v1/repos/Home-Network/home-network-k3s-impl` → `"private":false`, and `git ls-remote` returns `refs/heads/main`. |
|
||||
| Transport | **Plain HTTP, no TLS.** Accepted for the bring-up window only: LAN-local, and the repo holds no plaintext secrets (all SOPS-encrypted). |
|
||||
| Admin creds | `~/.config/home-k3s-bootstrap-gitea-admin.txt`, mode 0600, host-only, never committed. |
|
||||
|
||||
**THIS IS A SINGLE POINT OF FAILURE FOR PROD'S RECONCILIATION, and it is not even in a
|
||||
cluster.** It is one container on one workstation, with no HA, no backup of its own, and
|
||||
no restart guarantee beyond `--restart unless-stopped`. If that workstation is powered
|
||||
off, suspended, or the podman volume `gitea-bootstrap-data` is lost:
|
||||
- already-reconciled state **keeps running** — Flux does not tear anything down;
|
||||
- but **no new revision can land on prod**, and every `GitRepository` reconcile fails.
|
||||
So prod is *not* self-sufficient while this stands. Do not treat prod as production-ready
|
||||
on this footing, and do not let it become the steady state by inattention.
|
||||
|
||||
**One bring-up footgun:** `bootstrap-gitea.sh` pushes the operator's **current `HEAD`** to
|
||||
the bootstrap Gitea's `main`, whatever branch that HEAD is on. Before bootstrapping prod,
|
||||
confirm that remote `main` is the revision prod should run
|
||||
(`git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` vs local
|
||||
`main`). As of 2026-09-04 the remote `main` is the tip of the working branch, **not** local
|
||||
`main`.
|
||||
|
||||
**EXIT CONDITION (the cutover) — mirrors dev's own Phase-6 cutover, `docs/05`.**
|
||||
This item is not closable until all of it is done:
|
||||
- [ ] Gitea runs **on prod**, reconciled in-cluster, answering
|
||||
`gitea.home.keithjackson.dev` through prod's Gateway VIP with a publicly-trusted cert.
|
||||
- [ ] This repo is pushed/mirrored into that in-cluster Gitea.
|
||||
- [ ] The `clusters/prod` GitRepository url is changed to
|
||||
`https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`
|
||||
and the TEMPORARY comment block is **deleted** (dev's stanza is what "done" looks like).
|
||||
- [ ] `flux get sources git -n flux-system` on prod reconciles a revision from the new URL.
|
||||
- [ ] **Only then** stop the workstation container — kept stopped-not-deleted as recovery,
|
||||
exactly as dev retained its own bootstrap Gitea.
|
||||
- [x] **`roles/flux_bootstrap` could only bootstrap dev** — FIXED 2026-09-04. It invoked
|
||||
`bootstrap/flux/bootstrap.sh` without ever exporting `FLUX_ENV_PATH`, so the script fell
|
||||
through to its `clusters/dev` default and §8 step 4 (`--path=clusters/prod`) was
|
||||
unreachable without editing the role. Now parameterised on the standard group_vars seam:
|
||||
role default `flux_bootstrap_env_path: clusters/dev` (defaults/main.yml) is exported as
|
||||
`FLUX_ENV_PATH` by the role's task, and
|
||||
`inventory/group_vars/k3s_prod/vars.yml` overrides it to `clusters/prod` — the same
|
||||
mechanism as `kube_vip_address` / `k3s_data_dir` / `cluster_kubeconfig_name`. Dev has no
|
||||
override anywhere, so dev resolves `clusters/dev` exactly as before.
|
||||
- [ ] Confirm prod's backup target: dev's S3 (`versitygw` on the Proxmox host, `192.168.1.9`) is on
|
||||
the dev side of the network. Off-cluster backup is DR-critical (R-STATE-03).
|
||||
|
||||
@@ -1232,3 +1283,152 @@ single-server-etcd window, and avoids a later member-add dance.
|
||||
specifically.
|
||||
- [ ] Monitoring and alerts live on prod.
|
||||
- [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03).
|
||||
Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2.
|
||||
- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10):
|
||||
distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and
|
||||
prod's own Keycloak realm + per-service client secrets.
|
||||
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix**
|
||||
|
||||
### What happened
|
||||
|
||||
Prod's `external-dns` **deleted every one of dev's public DNS records.**
|
||||
|
||||
Both clusters ran `policy: sync` against the **same Cloudflare zone**, served the **same
|
||||
hostnames**, and — because neither overlay set `txtOwnerId` — both inherited the external-dns
|
||||
chart's default owner id **`default`**. ExternalDNS decides what it may delete by reading its own
|
||||
owner id back out of the TXT registry. Prod read dev's TXT entries, saw its own owner id, found no
|
||||
local resource claiming those names, and garbage-collected the records behind them. Dev recovered
|
||||
once prod was stopped. Prod's `platform` tier and everything downstream were then **suspended**, and
|
||||
remain suspended, so it cannot recur before this change is reviewed.
|
||||
|
||||
`policy: sync` was not the bug and has not been changed. What was missing was **identity**.
|
||||
|
||||
### The second bug, queued directly behind it
|
||||
|
||||
`platform-config` would have requested a `*.home.keithjackson.dev` wildcard `Certificate` from
|
||||
**letsencrypt-prod** — for a cert **dev already holds** — and both clusters would then have raced
|
||||
each other's `_acme-challenge` TXT records on that one zone. Neither would reliably solve DNS-01.
|
||||
This never fired only because the tier was suspended first.
|
||||
|
||||
### Root cause, stated precisely
|
||||
|
||||
Not external-dns. **Every hostname in the repo was a hardcoded literal.** `BASE_DOMAIN` existed in
|
||||
`clusters/base/cluster-vars.yaml` but was referenced by **no reconciled manifest** — the repo-derived
|
||||
status report flagged it as a dead knob. R-ENV-02 promises dev and prod differ only by `cluster-vars`
|
||||
values; for DNS and TLS that promise was false, so `clusters/prod` was *authored* to serve dev's
|
||||
public identity. Two clusters that describe the same DNS name will fight over it no matter how the
|
||||
DNS controller is configured.
|
||||
|
||||
### The architecture decision (operator)
|
||||
|
||||
```
|
||||
dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev
|
||||
prod -> *.prod.home.keithjackson.dev gitea.prod.home.keithjackson.dev
|
||||
apex -> CNAME <svc>.home.keithjackson.dev -> <svc>.prod.home.keithjackson.dev
|
||||
```
|
||||
|
||||
### PHASE 1 — **prod only. Dev did not move.** (this change)
|
||||
|
||||
`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it.
|
||||
|
||||
| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` |
|
||||
|---|---|---|---|
|
||||
| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` |
|
||||
| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` |
|
||||
| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` |
|
||||
|
||||
**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod
|
||||
alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken
|
||||
while the operator is away. Dev's rendered output is **byte-identical** except for one added line,
|
||||
`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified
|
||||
by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for
|
||||
both overlays and diffing.
|
||||
|
||||
**Two independent guards now, either sufficient alone:**
|
||||
|
||||
1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is
|
||||
the direct fix for the incident. Dev **must** render exactly `default`: its live records are
|
||||
already registered under that owner, and any other value would make dev's own external-dns stop
|
||||
recognising them and re-create duplicates beside the orphans.
|
||||
2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree.
|
||||
Defence in depth, added because the first guard is a single string in a single file.
|
||||
|
||||
The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a
|
||||
future environment that forgets the knob must not silently inherit dev's identity — that is exactly
|
||||
what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing.
|
||||
|
||||
### What could NOT be tokenized, and why
|
||||
|
||||
- **`OIDC_ISSUER_URL` / `OIDC_CALLBACK_URL` in `platform/headlamp/headlamp-oidc-credentials.sops.yaml`**
|
||||
— SOPS-encrypted (`encrypted_regex: ^(data|stringData)$`), so the two hostnames are ciphertext.
|
||||
Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and
|
||||
**cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake
|
||||
would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately.
|
||||
**Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak.
|
||||
Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client
|
||||
secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret
|
||||
is per-realm.
|
||||
- **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s
|
||||
**server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution
|
||||
can never reach it. Correctly tokenized on the **Ansible** side instead: new role default
|
||||
`k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in
|
||||
`inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`.
|
||||
`[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver
|
||||
validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the
|
||||
cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew).
|
||||
|
||||
### A pre-existing prod problem this incidentally FIXED
|
||||
|
||||
§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC
|
||||
issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN,
|
||||
making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on
|
||||
its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by
|
||||
**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer
|
||||
and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's
|
||||
external-dns has actually published the record; until then use the break-glass cert kubeconfig.
|
||||
|
||||
### Verify on prod's first reconcile
|
||||
|
||||
- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup
|
||||
as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone
|
||||
("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it
|
||||
will create **nothing** — fail-closed, never destructive. Fallback: set prod's
|
||||
`EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is
|
||||
sufficient on its own.
|
||||
- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching
|
||||
dev's `_acme-challenge` records.
|
||||
- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry
|
||||
entries still read owner `default` and that prod's read owner `prod`.
|
||||
- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client
|
||||
secret is realm-scoped; prod cannot reuse dev's).
|
||||
|
||||
### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**)
|
||||
|
||||
Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the
|
||||
Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change.
|
||||
Required, roughly in order:
|
||||
|
||||
1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the
|
||||
dev inventory group.
|
||||
2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover.
|
||||
3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client
|
||||
(`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`,
|
||||
`qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs
|
||||
updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a
|
||||
**rolling k3s restart** on the dev servers.
|
||||
4. **kubectl SSO re-issue** for every user (issuer changed).
|
||||
5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in
|
||||
`clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before*
|
||||
the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself.
|
||||
Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05).
|
||||
6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned
|
||||
records are for names dev no longer serves; retire them deliberately rather than letting a
|
||||
renamed owner orphan them.
|
||||
7. **Apex CNAMEs** `<svc>.home.keithjackson.dev` → `<svc>.prod.home.keithjackson.dev`. This is what
|
||||
actually **moves user traffic to prod**, and it must not land before prod serves those services.
|
||||
8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`,
|
||||
`dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`,
|
||||
`trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1
|
||||
— but they will fail against prod and must be parameterized before §9's "suite passes green
|
||||
against the prod cluster" can be claimed.
|
||||
+26
-5
@@ -79,11 +79,30 @@ Ansible installs Cilium; the Flux `platform/cilium` HelmRelease adopts config
|
||||
afterward. The parity-critical values **must match** between this role's
|
||||
`templates/cilium-values.yaml.j2` and that HelmRelease:
|
||||
- `kubeProxyReplacement: true`
|
||||
- `k8sServiceHost: {{ k3s_api_host }}` (= primary node IP, `10.73.0.11` by default)
|
||||
- `k8sServicePort: 6443`
|
||||
- `k8sServiceHost: {{ k3s_apiserver_endpoint }}` — **the kube-vip CONTROL-PLANE VIP**
|
||||
(`= kube_vip_address`: dev `192.168.1.49`, prod `192.168.2.9`), **not** `k3s_api_host`
|
||||
- `k8sServicePort: {{ k3s_api_port }}` (6443 in both environments)
|
||||
|
||||
The HelmRelease currently has `k8sServiceHost: "REPLACE-ME"` — WS-NET must set it
|
||||
to the same value this role uses (surfaced by `tofu output primary_node_ip`).
|
||||
### Two API addresses, split 2026-09-06 — do not re-merge them
|
||||
`k3s_api_host` and `k3s_apiserver_endpoint` were one variable, and the conflation put every
|
||||
prod `cilium-agent` on `192.168.2.10` (`k3s-prod-1`'s own address) on a cluster whose point is
|
||||
HA — verified live. Lose that node and the datapath keeps serving from Cilium's cache while no
|
||||
service or policy update ever lands, which is a silent failure, not an outage.
|
||||
|
||||
| variable | value | consumed by | why it must be that address |
|
||||
|---|---|---|---|
|
||||
| `k3s_api_host` | the founder's own node IP (dev `192.168.1.50`, prod `192.168.2.10`) | `roles/k3s_server`: the secondary `--server https://…:6443` join URL, and one `k3s_tls_sans` entry | on a **cold** build no VIP exists yet (kube-vip is a k3s auto-deploy manifest, so it needs a cluster first). The founder just ran `--cluster-init` and is up |
|
||||
| `k3s_apiserver_endpoint` | `{{ kube_vip_address }}` (dev `192.168.1.49`, prod `192.168.2.9`) | `roles/cilium` → `k8sServiceHost`; any steady-state client | it must survive losing any single server |
|
||||
|
||||
Consequences already handled: `playbooks/site.yml` imports `kube-vip.yml` **between** `k3s.yml`
|
||||
and `cilium.yml` (the VIP has to be floating before the CNI dials it), and `roles/kube_vip`'s
|
||||
last task blocks until the VIP answers on 6443. `k3s_tls_sans` already carried both addresses,
|
||||
so no cert reissue and no k3s restart are involved.
|
||||
|
||||
The Flux side of the same contract is `${K3S_API_HOST}` in `platform/cilium/helmrelease.yaml`,
|
||||
set per overlay in `clusters/<env>/kustomization.yaml`; `[parity]` means it must equal
|
||||
`k3s_apiserver_endpoint` for the SAME environment. Asserted by
|
||||
`conformance/tests/l0-static/ha-control-plane-as-code`.
|
||||
|
||||
## Interface assumed of `bootstrap/` (authored by other workers, NOT here)
|
||||
`roles/flux_bootstrap` invokes two entrypoints via overridable command vars:
|
||||
@@ -148,7 +167,9 @@ ansible-playbook playbooks/site.yml \
|
||||
What the roles then resolve **by themselves** for a prod host, from `group_vars/k3s_prod/vars.yml`:
|
||||
`k3s_primary_group=k3s_prod_primary` (the `--cluster-init` founder, the node-token `delegate_to`
|
||||
target, and the node Cilium fetches the kubeconfig from), `k3s_servers_group=k3s_prod` (the tls-san
|
||||
node-IP list), `k3s_api_host=192.168.2.10`, `kube_vip_address=192.168.2.9`, and
|
||||
node-IP list), `k3s_api_host=192.168.2.10` (the founder, bootstrap-join only),
|
||||
`kube_vip_address=192.168.2.9` — and therefore `k3s_apiserver_endpoint=192.168.2.9`, which is what
|
||||
Cilium dials — and
|
||||
`cluster_kubeconfig_name=prod.kubeconfig` — so a prod run can never overwrite the
|
||||
`.cache/dev.kubeconfig` that `infra/devcontainer/dev.sh` exports as `$KUBECONFIG`.
|
||||
|
||||
|
||||
@@ -27,19 +27,85 @@ k3s_servers_group: k3s_servers
|
||||
# a kubeconfig is a credential.)
|
||||
cluster_kubeconfig_name: dev.kubeconfig
|
||||
|
||||
# The kube-apiserver endpoint the whole stack agrees on: the primary's REAL node IP.
|
||||
# --- TWO API ADDRESSES, AND THEY ARE NOT INTERCHANGEABLE (split 2026-09-06) ----------
|
||||
# This used to be ONE variable, k3s_api_host, consumed by BOTH the cold-bootstrap join and
|
||||
# Cilium's k8sServiceHost. That conflation is what pointed every cilium-agent on the LIVE
|
||||
# prod cluster at 192.168.2.10 — k3s-prod-1's own address — on a cluster whose entire
|
||||
# purpose is HA: verified 2026-09-06, all three agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held 192.168.2.9. Lose that one node
|
||||
# and no service/policy update reaches the CNI until it returns (the existing datapath
|
||||
# survives on Cilium's cache, so it fails SILENTLY). The two consumers have genuinely
|
||||
# different requirements, so they now have two variables.
|
||||
#
|
||||
# k3s_api_host THE FOUNDER'S OWN ADDRESS. Bootstrap-time only.
|
||||
# k3s_apiserver_endpoint THE HA ENDPOINT (the kube-vip VIP). Steady-state.
|
||||
#
|
||||
# DO NOT "simplify" these back into one. See each block below for why each half must be
|
||||
# the address it is.
|
||||
|
||||
# THE FOUNDER'S OWN NODE IP — a BOOTSTRAP-TIME address, deliberately NOT the VIP.
|
||||
# NOTE: ansible_host is now the pfSense NAT SSH target (192.168.2.102:225x), NOT the node
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead. Used for k3s --tls-san /
|
||||
# secondary --server AND for Cilium k8sServiceHost so the Ansible bootstrap and the Flux
|
||||
# HelmRelease stay in PARITY (docs/02 contract).
|
||||
# IP — so we read the real 192.168.1.x from k3s_node_ip instead.
|
||||
#
|
||||
# EXACTLY TWO CONSUMERS, and both of them need the founder specifically:
|
||||
# 1. roles/k3s_server — the secondary `--server https://<host>:<port>` JOIN URL. On a COLD
|
||||
# build the VIP DOES NOT EXIST YET at join time: kube-vip is a k3s auto-deploy manifest
|
||||
# that only lands once a cluster exists, so a join aimed at the VIP is a join aimed at
|
||||
# an unanswered address and the cold build is unjoinable. The founder is up (it just ran
|
||||
# --cluster-init), which is the whole reason this variable is the founder.
|
||||
# 2. roles/k3s_server — one entry of k3s_tls_sans, so the founder's own address is a SAN
|
||||
# alongside the VIP and every other server's IP.
|
||||
#
|
||||
# ANYTHING THAT RUNS AGAINST A CLUSTER THAT ALREADY EXISTS MUST NOT USE THIS. It is a
|
||||
# single point of failure by construction. Use k3s_apiserver_endpoint below.
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['k3s_node_ip'] }}"
|
||||
|
||||
# Shared by both halves: k3s serves the API on 6443 in every environment, so the port is
|
||||
# not a per-environment knob and is not duplicated per address.
|
||||
k3s_api_port: 6443
|
||||
|
||||
# THE HA kube-apiserver ENDPOINT — a VIRTUAL address that reaches the cluster wherever the
|
||||
# control plane currently is, not a machine. OPERATOR DECISION 2026-09-06: in-cluster
|
||||
# components dial the VIP, as kube-vip was always intended to provide.
|
||||
#
|
||||
# DERIVED from kube_vip_address (below), never restated: an environment overrides the VIP in
|
||||
# ONE place (group_vars/k3s_prod/vars.yml sets kube_vip_address: 192.168.2.9) and this
|
||||
# follows automatically, so prod needs no override of its own and a fourth environment
|
||||
# cannot set one of the two and forget the other.
|
||||
#
|
||||
# NAMED FOR THE ROLE, NOT THE MECHANISM. Consumers want "the address that reaches the
|
||||
# apiserver, HA"; that it is currently floated by kube-vip is an implementation detail. If
|
||||
# the mechanism is ever something else (an external LB, a DNS name), this one line moves and
|
||||
# no consumer changes.
|
||||
#
|
||||
# CONSUMERS:
|
||||
# - roles/cilium -> cilium-values.yaml.j2 -> k8sServiceHost. [parity] with the Flux side:
|
||||
# clusters/base/cluster-vars.yaml K3S_API_HOST (and each overlay's patch) MUST equal
|
||||
# THIS value for that environment, or Flux and the bootstrap fight over the CNI's
|
||||
# control-plane path. See platform/cilium/helmrelease.yaml's PARITY CONTRACT header.
|
||||
#
|
||||
# TLS IS ALREADY COVERED — verified, not assumed: k3s_tls_sans (roles/k3s_server/defaults)
|
||||
# has always included kube_vip_address, and on 2026-09-06 the prod apiserver was reached
|
||||
# through https://192.168.2.9:6443 with FULL CA verification and tls-server-name=192.168.2.9
|
||||
# (infra/ansible/.cache/prod.kubeconfig, no insecure-skip-tls-verify). So the serving cert
|
||||
# needs no change and no k3s restart for this switch.
|
||||
#
|
||||
# ORDERING CONSEQUENCE, and it is why playbooks/site.yml changed too: Cilium now needs the
|
||||
# VIP to be UP before it installs. site.yml therefore imports kube-vip.yml BETWEEN k3s.yml
|
||||
# and cilium.yml, and roles/kube_vip waits for the VIP to answer before returning. kube-vip
|
||||
# can precede the CNI because its DaemonSet is hostNetwork with
|
||||
# KUBERNETES_SERVICE_HOST=127.0.0.1 — it needs neither a CNI nor kube-proxy.
|
||||
k3s_apiserver_endpoint: "{{ kube_vip_address }}"
|
||||
|
||||
# Control-plane HA VIP floated by kube-vip (roles/kube_vip). A stable failover
|
||||
# endpoint for the kube-apiserver, fronted by an operator pfSense port-forward at
|
||||
# 192.168.1.49:6443. Shared here so BOTH kube_vip AND the k3s_server tls-san list
|
||||
# agree on the address (the serving cert must carry this IP or TLS via the VIP
|
||||
# fails to validate).
|
||||
# 192.168.1.49:6443. Shared here so THREE things agree on the one address:
|
||||
# - roles/kube_vip — what to ARP and lease-elect for;
|
||||
# - roles/k3s_server — the tls-san entry (the serving cert must carry this IP or
|
||||
# TLS via the VIP fails to validate);
|
||||
# - k3s_apiserver_endpoint — above; what in-cluster components actually dial.
|
||||
# THIS is the one line an environment overrides (prod: 192.168.2.9, in
|
||||
# group_vars/k3s_prod/vars.yml); all three follow.
|
||||
kube_vip_address: "192.168.1.49"
|
||||
|
||||
# Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT
|
||||
|
||||
@@ -51,12 +51,18 @@ k3s_servers_group: k3s_prod
|
||||
# than no kubeconfig at all). .cache/ is gitignored; a kubeconfig is a credential.
|
||||
cluster_kubeconfig_name: prod.kubeconfig
|
||||
|
||||
# --- k3s API host — safe to set now, the primary's real IP is already known -------
|
||||
# --- k3s API host — THE FOUNDER'S ADDRESS, bootstrap-time only --------------------
|
||||
# Dev's k3s_api_host (group_vars/all.yml) reads a separate k3s_node_ip hostvar because
|
||||
# dev's ansible_host is the pfSense NAT SSH target, not the node's real address. Prod
|
||||
# has NO NAT hop — ansible_host in hosts.ini IS already the node's real, routable LAN
|
||||
# address (see the KEY DIFFERENCE note there) — so point straight at it instead of
|
||||
# needing a k3s_node_ip split.
|
||||
#
|
||||
# WHAT THIS IS *NOT*, since 2026-09-06: it is not the endpoint anything steady-state
|
||||
# dials. That is k3s_apiserver_endpoint (group_vars/all.yml), which derives from
|
||||
# kube_vip_address below and therefore needs NO override here. k3s_api_host is consumed
|
||||
# only by the secondary `--server` cold-join URL and by one tls-san entry. Read the
|
||||
# TWO API ADDRESSES block at the top of group_vars/all.yml before touching either.
|
||||
k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# cluster_controller_tls_server_name (group_vars/all.yml) derives from kube_vip_address
|
||||
# by template reference, so it needs no separate override here — it picks up the real
|
||||
@@ -75,8 +81,15 @@ k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}"
|
||||
# round (control-plane .49, ingress pool .53-.60).
|
||||
#
|
||||
# This is the address kube-vip floats for the API and that k3s tls-san's onto its
|
||||
# serving cert; secondaries use it as their --server join target. Dev's equivalent is
|
||||
# kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
# serving cert. Dev's equivalent is kube_vip_address: 192.168.1.49 in group_vars/all.yml.
|
||||
#
|
||||
# CORRECTION 2026-09-06: an earlier version of this comment claimed "secondaries use it as
|
||||
# their --server join target". They do not, and they must not — on a COLD build the VIP is
|
||||
# not up when the secondaries join (kube-vip is a k3s auto-deploy manifest and needs a
|
||||
# cluster first), so the join uses k3s_api_host, the founder. What DOES use this address is
|
||||
# k3s_apiserver_endpoint (group_vars/all.yml, derived from this line), i.e. Cilium's
|
||||
# k8sServiceHost and any other steady-state client. Overriding kube_vip_address here moves
|
||||
# BOTH the VIP and prod's Cilium endpoint together, which is the point of deriving it.
|
||||
#
|
||||
# HISTORY: an earlier pass here wrongly set this to 192.168.2.20 — that is the INGRESS
|
||||
# VIP, not the control-plane VIP. Corrected. Keep the two distinct.
|
||||
@@ -289,15 +302,58 @@ node_baseline_dns_servers:
|
||||
# LAN clients -> PiHole. Not a cluster concern.
|
||||
# nodes -> THIS FILE. Public resolvers, so the zone resolves PUBLICLY.
|
||||
#
|
||||
# THE ONE THING THAT BREAKS: the k3s API SERVER is a node process and carries
|
||||
# `oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab`
|
||||
# (roles/k3s_server/tasks/main.yml). It reaches the issuer through the NODE resolver, so on
|
||||
# prod it would get the PUBLIC record instead of the prod Gateway VIP — breaking kubectl
|
||||
# OIDC SSO. Dev avoids this because its node resolver is PiHole, which answers the zone
|
||||
# with the dev Gateway VIP.
|
||||
# THE k3s API SERVER is a node process and carries `oidc-issuer-url=...` (built from
|
||||
# k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through
|
||||
# these PUBLIC upstreams — not through CoreDNS's split-horizon answer.
|
||||
#
|
||||
# NOT blocking today (prod has no cluster and no Keycloak yet). Options, in
|
||||
# PROD-PROVISIONING.md §4: a narrow Ansible-managed /etc/hosts entry pinning
|
||||
# keycloak.home.keithjackson.dev -> 192.168.2.20 on prod nodes; or a local resolver on the
|
||||
# prod LAN serving the zone; or run prod on the cert kubeconfig and defer OIDC.
|
||||
# THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was
|
||||
# authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev
|
||||
# publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so
|
||||
# kubectl OIDC SSO on prod could not work at all. Now prod's issuer is
|
||||
# keycloak.prod.home.keithjackson.dev, and the public record for that name is published by
|
||||
# PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer
|
||||
# and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed.
|
||||
#
|
||||
# STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's
|
||||
# external-dns has actually published the record. Until then the issuer does not resolve
|
||||
# from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass
|
||||
# path, and it always works. Ordering, not a design gap.
|
||||
node_baseline_dns_search: []
|
||||
|
||||
# --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) --------------------
|
||||
# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/
|
||||
# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and
|
||||
# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a
|
||||
# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so
|
||||
# postBuild.substituteFrom never sees it. Full rationale in
|
||||
# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this.
|
||||
#
|
||||
# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the
|
||||
# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
|
||||
# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE);
|
||||
# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is
|
||||
# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client
|
||||
# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against.
|
||||
k3s_base_domain: "prod.home.keithjackson.dev"
|
||||
|
||||
# --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) -----
|
||||
# roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE
|
||||
# and group_vars/all.yml carries no override, so dev keeps resolving exactly that).
|
||||
# Prod bootstraps its OWN overlay. Before this seam existed the role could bootstrap
|
||||
# clusters/dev ONLY — bootstrap/flux/bootstrap.sh's FLUX_ENV_PATH default was never
|
||||
# exported by the role — so `flux bootstrap ... --path=clusters/prod` (§8 step 4 of
|
||||
# infra/PROD-PROVISIONING.md) was simply not reachable without editing the role.
|
||||
#
|
||||
# Same seam as kube_vip_address / k3s_api_host / k3s_data_dir / cluster_kubeconfig_name /
|
||||
# node_baseline_backports_kernel above: set once for the [k3s_prod] umbrella group, so a
|
||||
# FOURTH prod node inherits it from group membership with no per-host or per-run flag.
|
||||
#
|
||||
# WHAT clusters/prod STILL NEEDS before this is safe to run — do NOT invoke the
|
||||
# flux_bootstrap role until these are true (PROD-PROVISIONING.md §7):
|
||||
# - the remaining TBD-PROD-* placeholders in clusters/prod/kustomization.yaml are real
|
||||
# (S3_BACKUP_ENDPOINT); they are INVALID on purpose so an early apply fails loudly;
|
||||
# - the GitRepository url there is the TEMPORARY workstation Gitea
|
||||
# (http://192.168.2.132:3000/...) and that container is actually running and reachable
|
||||
# from the prod nodes — it is outside every cluster and can simply be powered off;
|
||||
# - the `sops-age` Secret is installed out of band on prod (bootstrap/sops).
|
||||
flux_bootstrap_env_path: "clusters/prod"
|
||||
@@ -3,6 +3,13 @@
|
||||
# Control-plane ONLY (svc_enable=false) — Cilium owns Service LoadBalancing.
|
||||
# Writes the manifest into each server's k3s auto-deploy dir; k3s applies it.
|
||||
#
|
||||
# PART OF THE ONE-COMMAND BRING-UP SINCE 2026-09-06, not a separate operator step:
|
||||
# playbooks/site.yml imports this play BETWEEN k3s.yml and cilium.yml, because Cilium's
|
||||
# k8sServiceHost is now this VIP (k3s_apiserver_endpoint, group_vars/all.yml) and the CNI
|
||||
# cannot be installed against an endpoint nothing announces yet. The role's final task waits
|
||||
# for the VIP to answer on 6443, so a failure to float it stops the run here with a named
|
||||
# cause instead of surfacing later as unreachable cilium-agents.
|
||||
#
|
||||
# TARGET (R-ENV-02): `k3s_hosts` defaults to dev's `k3s_servers`; prod is
|
||||
# ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod
|
||||
# (the VIP itself comes from kube_vip_address, already scoped per environment in
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
---
|
||||
# Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER
|
||||
# `tofu apply` has created and cloud-init has baselined the VMs):
|
||||
# k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off.
|
||||
# k3s HA install -> kube-vip control-plane VIP -> Cilium CNI -> verify readiness
|
||||
# -> Flux + SOPS hand-off.
|
||||
# (kube-vip joined the chain 2026-09-06 — see the note at its import below; it is a
|
||||
# prerequisite of the CNI now, not a separate operator step.)
|
||||
#
|
||||
# TARGET (R-ENV-02): every imported play takes its host pattern from a var that
|
||||
# defaults to the dev group, so `ansible-playbook playbooks/site.yml` with no extra
|
||||
@@ -20,6 +23,29 @@
|
||||
- name: Import - install k3s HA control plane
|
||||
ansible.builtin.import_playbook: k3s.yml
|
||||
|
||||
# ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and
|
||||
# Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint =
|
||||
# kube_vip_address) rather than the founder's own node IP, so installing Cilium before the
|
||||
# VIP is announced would point every agent at an address nothing answers on — the exact
|
||||
# shape of the 2026-09-01 incident, where agents that could not reach the API server
|
||||
# deadlocked on a cold cluster.
|
||||
#
|
||||
# IT CANNOT COME EARLIER. kube-vip ships as a k3s auto-deploy manifest
|
||||
# (<data-dir>/server/manifests), so a cluster must exist before it can be applied — hence
|
||||
# after k3s.yml, not before.
|
||||
#
|
||||
# IT DOES NOT NEED THE CNI, which is what makes this ordering possible at all: the kube-vip
|
||||
# DaemonSet is hostNetwork with KUBERNETES_SERVICE_HOST=127.0.0.1 (roles/kube_vip's template
|
||||
# explains why), so it reaches the node-local apiserver with no CNI and no kube-proxy, and
|
||||
# DaemonSet pods carry the default not-ready/network-unavailable tolerations that let them
|
||||
# schedule onto a still-NotReady node. roles/kube_vip blocks until the VIP actually answers
|
||||
# on 6443, so cilium.yml below cannot start against a VIP that is not up.
|
||||
#
|
||||
# Re-running this against the LIVE dev cluster is a no-op for this play: dev already runs
|
||||
# kube-vip with the same rendered manifest, and the wait finds the VIP already answering.
|
||||
- name: Import - float the control-plane HA VIP (kube-vip) BEFORE the CNI dials it
|
||||
ansible.builtin.import_playbook: kube-vip.yml
|
||||
|
||||
- name: Import - install Cilium CNI
|
||||
ansible.builtin.import_playbook: cilium.yml
|
||||
|
||||
|
||||
@@ -2,7 +2,17 @@
|
||||
# cilium role defaults (ADR-0002, docs/02 Cilium-ownership contract).
|
||||
# This role performs the INITIAL Cilium install; the Flux platform/cilium
|
||||
# HelmRelease adopts config afterward. The parity-critical values
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from k3s_api_host.
|
||||
# (kubeProxyReplacement, k8sServiceHost/Port) are templated from
|
||||
# k3s_apiserver_endpoint / k3s_api_port (group_vars/all.yml).
|
||||
#
|
||||
# k8sServiceHost IS THE HA VIP, NOT THE FOUNDER (changed 2026-09-06). It reads
|
||||
# k3s_apiserver_endpoint — which derives from kube_vip_address, so it is dev 192.168.1.49 /
|
||||
# prod 192.168.2.9 — and deliberately NOT k3s_api_host, the founder's own node IP. Those two
|
||||
# were one variable until 2026-09-06, and the conflation is what left all three prod
|
||||
# cilium-agents dialing k3s-prod-1 on an HA cluster. Do not re-merge them.
|
||||
#
|
||||
# THIS ROLE THEREFORE DEPENDS ON THE VIP EXISTING. playbooks/site.yml runs kube-vip.yml
|
||||
# before cilium.yml and roles/kube_vip blocks until the VIP answers on 6443.
|
||||
|
||||
# Exact pin — MUST equal the platform/cilium HelmRelease chart.spec.version (1.20.1, an
|
||||
# exact pin, not a range). Renovate bumps both together.
|
||||
@@ -15,9 +25,20 @@ cilium_namespace: "kube-system"
|
||||
# Cilium 1.20.1 targets Gateway API v1.6.1.
|
||||
gateway_api_version: "v1.6.1"
|
||||
gateway_api_crd_urls:
|
||||
# ONE URL, deliberately. As of v1.6.0 the STANDARD channel bundle already contains
|
||||
# TLSRoute (plus TCPRoute, UDPRoute, ListenerSet and BackendTLSPolicy), so the separate
|
||||
# experimental TLSRoute manifest this list used to carry is redundant — and worse than
|
||||
# redundant: v1.6 also ships a ValidatingAdmissionPolicy,
|
||||
# `safe-upgrades.gateway.networking.k8s.io`, which REJECTS an experimental-channel CRD
|
||||
# applied over a standard-channel one. Applying the old second URL therefore fails hard:
|
||||
# ValidatingAdmissionPolicy 'safe-upgrades.gateway.networking.k8s.io' ... denied request:
|
||||
# Installing experimental CRDs on top of standard channel CRDs is prohibited by default.
|
||||
# Verified 2026-09-04 on the prod bootstrap: standard-install.yaml v1.6.1 alone produces the
|
||||
# exact 10 gateway.networking.k8s.io CRDs the LIVE dev cluster is running, tlsroutes included
|
||||
# and annotated channel=standard, bundle-version=v1.6.1 — so this is parity with dev, not a
|
||||
# reduction. Dev needs no reapply; it already has them. If gateway_api_version is ever moved
|
||||
# BACK below v1.6.0, the experimental TLSRoute URL has to come back with it.
|
||||
- "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{ gateway_api_version }}/standard-install.yaml"
|
||||
# TLSRoute (experimental) is required by Cilium's Gateway API support.
|
||||
- "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{ gateway_api_version }}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
|
||||
|
||||
# Local kubeconfig fetched from the primary. *.kubeconfig is gitignored. The FILENAME
|
||||
# comes from cluster_kubeconfig_name (group_vars) so a prod run cannot overwrite the dev
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
run_once: true
|
||||
changed_when: false
|
||||
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_api_host)
|
||||
- name: Render the Cilium values file (parity-critical settings from k3s_apiserver_endpoint)
|
||||
ansible.builtin.template:
|
||||
src: cilium-values.yaml.j2
|
||||
dest: "{{ cilium_values_path }}"
|
||||
|
||||
@@ -4,9 +4,24 @@
|
||||
---
|
||||
# [parity] k3s kube-proxy is disabled → Cilium replaces it (ADR-0001/0002).
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = the k3s API endpoint (primary node IP). The Flux HelmRelease's
|
||||
# k8sServiceHost "REPLACE-ME" must be set to this same value.
|
||||
k8sServiceHost: "{{ k3s_api_host }}"
|
||||
# [parity] = k3s_apiserver_endpoint, THE HA CONTROL-PLANE VIP (kube_vip_address) — dev
|
||||
# 192.168.1.49, prod 192.168.2.9. NOT k3s_api_host, which is the FOUNDER'S OWN node IP:
|
||||
# pointing the CNI's kube-proxy replacement at one machine on a 3-server HA cluster means
|
||||
# losing that machine stops every service/policy update reaching the datapath, silently
|
||||
# (the existing datapath keeps working from Cilium's cache, so nothing looks broken). That
|
||||
# was the live state of prod until 2026-09-06 — all three cilium-agents carried
|
||||
# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held .9. See group_vars/all.yml.
|
||||
#
|
||||
# [parity] The Flux side is platform/cilium/helmrelease.yaml's k8sServiceHost:
|
||||
# "${K3S_API_HOST}", whose per-environment value comes from clusters/<env>/kustomization.yaml.
|
||||
# THAT VALUE MUST EQUAL k3s_apiserver_endpoint for the SAME environment. If it does not,
|
||||
# Flux and this bootstrap render different control-plane paths and fight over the CNI.
|
||||
#
|
||||
# THE VIP MUST BE UP BEFORE THIS ROLE RUNS. playbooks/site.yml imports kube-vip.yml between
|
||||
# k3s.yml and cilium.yml for exactly this reason, and roles/kube_vip blocks until the VIP
|
||||
# answers. Rendering this against a VIP that nothing announces yet reproduces the
|
||||
# 2026-09-01 shape: agents that cannot reach the API server on a cold cluster.
|
||||
k8sServiceHost: "{{ k3s_apiserver_endpoint }}"
|
||||
k8sServicePort: {{ k3s_api_port }}
|
||||
|
||||
# Ingress via Gateway API — no Traefik (docs/09, ADR-0002).
|
||||
|
||||
@@ -19,5 +19,22 @@ flux_bootstrap_repo_root: "{{ playbook_dir }}/../../.."
|
||||
flux_bootstrap_sops_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/sops/install-age-key.sh"
|
||||
|
||||
# Assumed interface of bootstrap/flux (GitOps Engineer, T-1.4):
|
||||
# runs `flux bootstrap ... --path=clusters/dev` against this cluster.
|
||||
# runs `flux install` + `kubectl apply -k <env path>` against this cluster.
|
||||
flux_bootstrap_flux_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/flux/bootstrap.sh"
|
||||
|
||||
# --- WHICH ENVIRONMENT OVERLAY THIS BOOTSTRAPS (R-ENV-02) -------------------------
|
||||
# The one thing that makes this role environment-agnostic. bootstrap/flux/bootstrap.sh
|
||||
# reads FLUX_ENV_PATH (default clusters/dev) and ends with `kubectl apply -k <that>`;
|
||||
# tasks/main.yml exports this value into that variable, so the role NEVER names an
|
||||
# environment and the task graph is identical for dev and prod.
|
||||
#
|
||||
# THE DEFAULT IS DEV ON PURPOSE — dev is LIVE, and group_vars/all.yml carries no
|
||||
# override, so a dev run resolves exactly the value it always had (clusters/dev) and
|
||||
# behaves byte-for-byte as before.
|
||||
#
|
||||
# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same group_vars
|
||||
# seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name /
|
||||
# node_baseline_backports_kernel. No role fork, no per-run extra-var, no second
|
||||
# playbook: group membership decides. Before this existed the role could bootstrap
|
||||
# ONLY clusters/dev, so prod bring-up was impossible without editing the role.
|
||||
flux_bootstrap_env_path: "clusters/dev"
|
||||
@@ -4,11 +4,17 @@
|
||||
# — it calls the entrypoints the Security + GitOps Engineers author there.
|
||||
#
|
||||
# Order matters: install the age decryption key FIRST (so Flux can decrypt on its
|
||||
# very first reconcile), THEN run `flux bootstrap` at clusters/dev.
|
||||
# very first reconcile), THEN bootstrap Flux at the environment's overlay.
|
||||
#
|
||||
# The two commands are variables (see defaults/main.yml). If the bootstrap workers
|
||||
# expose different entrypoints, override flux_bootstrap_sops_cmd / _flux_cmd — the
|
||||
# task graph does not change.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): this role names NO environment. Which overlay gets
|
||||
# bootstrapped comes from `flux_bootstrap_env_path` (defaults/main.yml, default
|
||||
# clusters/dev), exported below as FLUX_ENV_PATH — the variable bootstrap/flux/
|
||||
# bootstrap.sh already reads. Prod overrides it in group_vars/k3s_prod/vars.yml, the
|
||||
# same seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name.
|
||||
|
||||
- name: Assert the bootstrap/ subtree exists (authored by the other worker)
|
||||
ansible.builtin.stat:
|
||||
@@ -29,11 +35,14 @@
|
||||
run_once: true
|
||||
changed_when: true
|
||||
|
||||
- name: Bootstrap Flux against clusters/dev (bootstrap/flux)
|
||||
- name: Bootstrap Flux against this environment's overlay (bootstrap/flux)
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ flux_bootstrap_flux_cmd }}"
|
||||
environment:
|
||||
KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}"
|
||||
# The ONLY per-environment input. clusters/dev by default (dev unchanged);
|
||||
# clusters/prod for [k3s_prod] hosts via group_vars.
|
||||
FLUX_ENV_PATH: "{{ flux_bootstrap_env_path }}"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
|
||||
@@ -59,6 +59,17 @@ k3s_server_mode: "secondary"
|
||||
# serving cert on restart when this set changes. Deduplicated (the primary IP is
|
||||
# also k3s_api_host).
|
||||
#
|
||||
# THIS LIST COVERS BOTH API ADDRESSES, WHICH IS WHY THE 2026-09-06 SPLIT NEEDED NO CERT
|
||||
# CHANGE AND NO k3s RESTART. group_vars/all.yml now distinguishes k3s_api_host (the
|
||||
# founder, for the cold join) from k3s_apiserver_endpoint (= kube_vip_address, what Cilium
|
||||
# and every steady-state client dial). Both are already in this set, so repointing a
|
||||
# consumer from one to the other cannot produce a TLS failure. VERIFIED LIVE 2026-09-06,
|
||||
# not assumed: prod's apiserver answered through https://192.168.2.9:6443 with full CA
|
||||
# verification and tls-server-name=192.168.2.9. Do not remove either entry — dropping
|
||||
# kube_vip_address breaks every client of the VIP; dropping k3s_api_host breaks nothing
|
||||
# today only because it is also in the k3s_servers_group node-IP list below, and that
|
||||
# coincidence is not a contract.
|
||||
#
|
||||
# k3s_servers_group — NOT a literal group name — is what makes this role environment-
|
||||
# agnostic (R-ENV-02): group_vars/all.yml sets it to `k3s_servers` (dev) and
|
||||
# group_vars/k3s_prod/vars.yml to `k3s_prod`, so each environment SANs its own servers.
|
||||
@@ -66,3 +77,33 @@ k3s_tls_sans: >-
|
||||
{{ ([k3s_api_host, kube_vip_address, '127.0.0.1']
|
||||
+ (groups[k3s_servers_group] | map('extract', hostvars, 'k3s_node_ip') | list))
|
||||
| unique }}
|
||||
|
||||
# --- Keycloak OIDC issuer for kubectl SSO (R-IDP-03, R-ENV-02) -----------------------
|
||||
# THE ANSIBLE-SIDE HALF OF ${BASE_DOMAIN}, and it cannot be the Flux-side half.
|
||||
#
|
||||
# The kube-apiserver's `oidc-issuer-url` is a k3s SERVER FLAG written into
|
||||
# /etc/rancher/k3s/config.yaml by this role — it is NOT a Kubernetes manifest, so Flux never
|
||||
# sees it and `postBuild.substituteFrom: cluster-vars` cannot reach it. The platform hostname
|
||||
# suffix therefore exists as ONE FACT IN TWO PLACES, and they must agree:
|
||||
#
|
||||
# Flux side clusters/base/cluster-vars.yaml BASE_DOMAIN (per-overlay)
|
||||
# Ansible side THIS variable k3s_base_domain (per-inventory-group)
|
||||
#
|
||||
# [parity] k3s_base_domain MUST equal the BASE_DOMAIN the SAME cluster's Flux overlay sets.
|
||||
# If they disagree, the apiserver validates ID tokens against a DIFFERENT Keycloak than the
|
||||
# one the cluster actually runs: every `kubectl` SSO login fails with an issuer mismatch,
|
||||
# while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken",
|
||||
# not like a config skew. The break-glass cert kubeconfig is why this is recoverable.
|
||||
#
|
||||
# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same
|
||||
# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the
|
||||
# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte
|
||||
# what it already is and the Restart handler does NOT fire on a re-run.
|
||||
#
|
||||
# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam
|
||||
# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it
|
||||
# from group membership rather than from anyone remembering.
|
||||
k3s_base_domain: "home.keithjackson.dev"
|
||||
|
||||
# Derived, so the realm path is stated once. Override k3s_base_domain, not this.
|
||||
k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab"
|
||||
@@ -75,13 +75,19 @@
|
||||
# OIDC (Keycloak SSO for kubectl). The API server trusts ID tokens from the homelab realm;
|
||||
# the `kubernetes` PUBLIC client (PKCE) + kubelogin mint them, and RBAC binds the `groups`
|
||||
# claim (platform/keycloak/k8s-oidc-rbac.yaml: oidc:platform-admins->cluster-admin,
|
||||
# oidc:viewers->view). Nodes resolve keycloak.home.keithjackson.dev to the Cilium gateway VIP
|
||||
# (192.168.1.53) and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:`
|
||||
# oidc:viewers->view). Nodes resolve keycloak.<k3s_base_domain> to the Cilium gateway VIP
|
||||
# and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:`
|
||||
# prefixes namespace federated identities away from cert / service-account subjects.
|
||||
# ADDITIVE + break-glass-safe: the cert-based admin kubeconfig keeps full access with or
|
||||
# without Keycloak, so a bad realm/OIDC state can never lock you out of the cluster.
|
||||
#
|
||||
# NOT a Flux ${BASE_DOMAIN} token: this is a k3s SERVER FLAG, not a manifest, so
|
||||
# postBuild.substituteFrom never touches it. k3s_oidc_issuer_url derives from
|
||||
# k3s_base_domain (defaults/main.yml), whose [parity] contract with the cluster's
|
||||
# BASE_DOMAIN is documented there. Dev's default renders the value it already runs, so
|
||||
# config.yaml is unchanged and the Restart handler does not fire.
|
||||
kube-apiserver-arg:
|
||||
- "oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab"
|
||||
- "oidc-issuer-url={{ k3s_oidc_issuer_url }}"
|
||||
- "oidc-client-id=kubernetes"
|
||||
- "oidc-username-claim=email"
|
||||
- "oidc-username-prefix=oidc:"
|
||||
@@ -98,15 +104,20 @@
|
||||
INSTALL_K3S_VERSION: "{{ k3s_version }}"
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
# PATH FOLLOWS --data-dir. k3s writes the node-token to <data-dir>/server/token, NOT to
|
||||
# the literal /var/lib/rancher/k3s. On dev k3s_data_dir IS /var/lib/rancher/k3s (the role
|
||||
# default, restated explicitly above) so this is byte-for-byte the old path; on prod
|
||||
# --data-dir is /data/k3s (group_vars/k3s_prod) and the hardcoded path did not exist at all
|
||||
# — this task would have timed out after 180s and the two secondaries could never join.
|
||||
- name: Wait for the k3s node-token to exist (primary)
|
||||
ansible.builtin.wait_for:
|
||||
path: /var/lib/rancher/k3s/server/token
|
||||
path: "{{ k3s_data_dir }}/server/token"
|
||||
timeout: 180
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
- name: Read the k3s node-token (primary)
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/token
|
||||
src: "{{ k3s_data_dir }}/server/token"
|
||||
register: k3s_token_raw
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
@@ -126,9 +137,11 @@
|
||||
# "The primary" is resolved through k3s_primary_group (group_vars: `k3s_primary` for
|
||||
# dev, `k3s_prod_primary` for prod), never a literal group name — so a prod secondary
|
||||
# delegates to the PROD primary using this same role, unchanged (R-ENV-02).
|
||||
# k3s_data_dir is a CLUSTER-WIDE value (group_vars for prod, role default for dev), so the
|
||||
# secondary's own copy of it is also the primary's data dir — correct for the delegated read.
|
||||
- name: Fetch the k3s node-token directly from the primary (secondary)
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/token
|
||||
src: "{{ k3s_data_dir }}/server/token"
|
||||
delegate_to: "{{ groups[k3s_primary_group][0] }}"
|
||||
register: k3s_primary_token_raw
|
||||
when:
|
||||
@@ -142,6 +155,18 @@
|
||||
| default(k3s_primary_token_raw.content | b64decode | trim, true) }}
|
||||
when: k3s_server_mode == 'secondary'
|
||||
|
||||
# THE JOIN TARGET IS k3s_api_host — THE FOUNDER — AND MUST NOT BECOME THE VIP.
|
||||
# group_vars/all.yml splits the two API addresses (k3s_api_host = the founder's own node IP,
|
||||
# k3s_apiserver_endpoint = the kube-vip HA VIP) and this is the canonical founder-only
|
||||
# consumer. On a COLD build the VIP does not exist at this point in the sequence: kube-vip
|
||||
# is a k3s auto-deploy manifest, so it cannot be applied until a cluster exists, and
|
||||
# playbooks/site.yml only imports kube-vip.yml AFTER k3s.yml. A `--server` pointed at the
|
||||
# VIP would therefore dial an address nothing announces and the cold build would be
|
||||
# unjoinable — with no diagnostic beyond a connection timeout.
|
||||
# On a WARM single-node rejoin (`--limit k3s-dev-N`) the founder is up and answering too, so
|
||||
# the founder works in both cases and the VIP works in only one. Cilium's k8sServiceHost is
|
||||
# the opposite trade (steady state, must survive losing the founder) and uses the endpoint
|
||||
# variable instead.
|
||||
- name: Install k3s secondary (join as server)
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
|
||||
@@ -24,6 +24,16 @@ kube_vip_port: 6443
|
||||
# only if detection is wrong.
|
||||
kube_vip_interface: ""
|
||||
|
||||
# How long to wait for the VIP to actually ANSWER on kube_vip_port after the manifest is
|
||||
# written (seconds). This is a real gate, not cosmetics: since 2026-09-06 Cilium's
|
||||
# k8sServiceHost is this VIP (k3s_apiserver_endpoint), and playbooks/site.yml runs this role
|
||||
# immediately before cilium.yml — so if the VIP never comes up, the run must stop HERE with
|
||||
# a clear message rather than proceed to install a CNI pointed at an address nothing
|
||||
# answers. On a cold cluster the budget covers k3s's deploy controller applying the
|
||||
# manifest, the ghcr.io image pull and leader election; on an already-running cluster the
|
||||
# check returns immediately.
|
||||
kube_vip_wait_timeout: 300
|
||||
|
||||
# Leader-election tuning (seconds) — a snappy failover without thrashing.
|
||||
kube_vip_lease_duration: 15
|
||||
kube_vip_renew_deadline: 10
|
||||
@@ -31,4 +41,14 @@ kube_vip_retry_period: 2
|
||||
|
||||
# k3s auto-deploy manifests dir: k3s applies anything here on start, so kube-vip
|
||||
# is reproducible and independent of Flux/Helm.
|
||||
kube_vip_manifest_path: "/var/lib/rancher/k3s/server/manifests/kube-vip.yaml"
|
||||
#
|
||||
# THE PATH FOLLOWS --data-dir. k3s reads auto-deploy manifests from
|
||||
# <data-dir>/server/manifests, so this must be derived from the cluster-wide
|
||||
# k3s_data_dir (roles/k3s_server turns the same variable into --data-dir), not from the
|
||||
# literal /var/lib/rancher/k3s. Dev leaves k3s_data_dir unset outside roles/k3s_server's
|
||||
# own defaults, so the fallback below reproduces dev's existing path exactly; prod sets
|
||||
# k3s_data_dir: /data/k3s in group_vars/k3s_prod, where the old hardcoded path was simply
|
||||
# not the directory k3s watches — the manifest would have been written and silently never
|
||||
# applied, and the control-plane VIP would never have come up.
|
||||
kube_vip_manifest_path: >-
|
||||
{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}/server/manifests/kube-vip.yaml
|
||||
@@ -1,17 +1,34 @@
|
||||
---
|
||||
# Deploy kube-vip (control-plane HA VIP) via the k3s auto-deploy manifests dir.
|
||||
# k3s applies /var/lib/rancher/k3s/server/manifests/*.yaml on start, so this is
|
||||
# reproducible and independent of Flux. The same manifest is written on every
|
||||
# k3s applies <data-dir>/server/manifests/*.yaml on start, so this is
|
||||
# reproducible and independent of Flux (see kube_vip_manifest_path in defaults/main.yml —
|
||||
# the dir follows k3s's --data-dir, which prod moves to /data/k3s). The same manifest is written on every
|
||||
# server so the DaemonSet survives the loss of any single node.
|
||||
|
||||
- name: Detect the LAN interface holding the node's 192.168.1.5x address
|
||||
# WHICH INTERFACE kube-vip ARPs THE VIP ON. Detected from this node's OWN k3s_node_ip
|
||||
# (inventory, per host) rather than a hardcoded subnet: the previous awk matched the
|
||||
# literal 192.168.1.5[0-2], i.e. DEV's node addresses, so on prod (192.168.2.10/.11/.12)
|
||||
# it returned nothing and the assert below aborted the play. k3s_node_ip is the same
|
||||
# address k3s advertises with --node-ip and is defined for dev and prod alike, so this
|
||||
# resolves eno1 on prod and the identical dev interface it always resolved before.
|
||||
# `ip addr show to <ip>/32` does the matching in iproute2 instead of a regex over dotted
|
||||
# quads (where `.` would match any character).
|
||||
- name: Detect the LAN interface holding this node's k3s_node_ip
|
||||
ansible.builtin.shell: >-
|
||||
set -o pipefail;
|
||||
ip -o -4 addr show | awk '/192\.168\.1\.5[0-2]\//{print $2; exit}'
|
||||
ip -o -4 addr show to {{ k3s_node_ip }}/32 | awk '{print $2; exit}'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: kube_vip_iface_detect
|
||||
changed_when: false
|
||||
# RUN THIS EVEN UNDER --check. It is `ip -o -4 addr show`: it reads and changes nothing.
|
||||
# Without check_mode: false the shell module is SKIPPED in check mode, kube_vip_iface_detect
|
||||
# has no stdout, and the assert two tasks down aborts the play with "Could not detect the
|
||||
# interface ... set kube_vip_interface explicitly" — a misleading failure that says nothing
|
||||
# about the node. That mattered less while this play was an out-of-band operator step; it
|
||||
# matters now that playbooks/site.yml imports it, because `--check` on the bring-up before a
|
||||
# cold prod build is exactly the rehearsal an operator should be able to run.
|
||||
check_mode: false
|
||||
|
||||
- name: Resolve the kube-vip VIP interface (override wins, else detected)
|
||||
ansible.builtin.set_fact:
|
||||
@@ -25,7 +42,7 @@
|
||||
- kube_vip_vip_interface is defined
|
||||
- kube_vip_vip_interface | length > 0
|
||||
fail_msg: >-
|
||||
Could not detect the 192.168.1.5x LAN interface for kube-vip on
|
||||
Could not detect the interface holding {{ k3s_node_ip }} for kube-vip on
|
||||
{{ inventory_hostname }} — set kube_vip_interface explicitly.
|
||||
|
||||
- name: Ensure the k3s auto-deploy manifests directory exists
|
||||
@@ -43,3 +60,34 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
# PROVE THE VIP IS ACTUALLY UP BEFORE THIS PLAY RETURNS (added 2026-09-06).
|
||||
#
|
||||
# Templating the manifest is not the same as having a floating endpoint: k3s's deploy
|
||||
# controller still has to apply it, the image still has to pull, and leader election still
|
||||
# has to pick an owner. Nothing used to check, which was tolerable while the VIP was only an
|
||||
# operator convenience. It is not tolerable now — Cilium's k8sServiceHost IS this VIP
|
||||
# (k3s_apiserver_endpoint) and playbooks/site.yml runs cilium.yml straight after this play,
|
||||
# so an unannounced VIP would mean installing a CNI that cannot reach the API server. That
|
||||
# failure is the 2026-09-01 shape and it is very expensive to diagnose from the far side;
|
||||
# failing here instead names the actual cause.
|
||||
#
|
||||
# run_once: the VIP is ONE address for the whole environment, so one probe answers for the
|
||||
# environment. Probing from a server host (not the controller) is deliberate: the servers
|
||||
# are on the VIP's own LAN in both environments, whereas the dev controller reaches dev's
|
||||
# API only through a pfSense NAT — a controller-side probe would have been a property of
|
||||
# where the play was launched from. READ-ONLY: it opens a TCP connection and changes nothing.
|
||||
- name: Wait for the control-plane VIP to answer (the CNI's endpoint depends on it)
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ kube_vip_address }}"
|
||||
port: "{{ kube_vip_port }}"
|
||||
timeout: "{{ kube_vip_wait_timeout }}"
|
||||
run_once: true
|
||||
changed_when: false
|
||||
vars:
|
||||
ansible_become: false
|
||||
# DELIBERATELY NOT check_mode: false, unlike the interface probe above — so a `--check`
|
||||
# rehearsal SKIPS this and does not sit on a socket for kube_vip_wait_timeout. wait_for
|
||||
# declares no check-mode support, so the skip is the module's own behaviour and this comment
|
||||
# is here only so the skip does not read as an oversight. CONSEQUENCE, stated plainly: a
|
||||
# `--check` run does NOT verify the VIP is up. Only a real run gates the CNI on it.
|
||||
@@ -183,6 +183,29 @@ node_baseline_protected_mounts:
|
||||
- /home
|
||||
- /tmp
|
||||
|
||||
# --- WIRELESS (tasks/main.yml, tag `wireless`) ---------------------------------------
|
||||
# A CLUSTER NODE GETS EXACTLY ONE PATH ONTO THE NETWORK. ON by default — unlike the
|
||||
# backports kernel above, which is a prod-only deviation, this is a cluster-wide policy and
|
||||
# it is a provable NO-OP where it does not apply: the tasks act only on interfaces the node
|
||||
# reports as real 802.11 devices (a `phy80211` link in sysfs), and dev's Proxmox VMs have
|
||||
# none, so on dev nothing is written, nothing is unloaded and nothing changes. Defaulting it
|
||||
# true is what makes a FOURTH node with a wifi card inherit the policy from being a cluster
|
||||
# node, rather than from someone remembering to set a flag.
|
||||
#
|
||||
# WHY IT EXISTS: k3s-prod-3 came up on both eno1 (192.168.2.12, the inventory address) and
|
||||
# wlp1s0 (192.168.2.195, DHCP on the same /24). The wifi lease's dhcpcd run rewrote
|
||||
# /etc/resolv.conf three minutes after this role wrote it, and a second address on the LAN
|
||||
# where kube-vip floats 192.168.2.9 and Cilium L2-announces .20-.30 is a problem in its own
|
||||
# right. See the `wireless` block in tasks/main.yml for the full reasoning, including why the
|
||||
# DRIVER MODULE is the layer that gets blocked rather than the link, a service, or rfkill.
|
||||
#
|
||||
# SET IT FALSE for a host that must genuinely keep wifi (there is none today). That does NOT
|
||||
# undo an already-applied block — delete the file below on the node as well.
|
||||
node_baseline_disable_wireless: true
|
||||
# Where the persistent block lives. 99- so it is read last and overrides any earlier
|
||||
# permissive rule; the name says what it does, on a node with no repo checkout to hand.
|
||||
node_baseline_wireless_modprobe_file: /etc/modprobe.d/99-k3s-no-wireless.conf
|
||||
|
||||
# --- DNS resilience (R-DNS-01 / R-DNS-03) -------------------------------------------
|
||||
# These are the DEV values. Prod overrides them in inventory/group_vars/k3s_prod/vars.yml
|
||||
# (the same seam kube_vip_address / k3s_api_host / cluster_kubeconfig_name use) — NOT in
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
# TAGS — every task here carries one of `packages`, `kernel`, `system`, `disks` or `dns`. An
|
||||
# untagged run is unchanged (all five run, in this order). The split exists because the PACKAGE floor
|
||||
# TAGS — every task here carries one of `packages`, `kernel`, `system`, `wireless`, `disks` or
|
||||
# `dns`. An untagged run is unchanged (all six run, in this order). The split exists because the PACKAGE floor
|
||||
# and the DATA-DISK work have different readiness: packages are safe on any node from
|
||||
# first contact, while the disk tasks need block devices that a new environment may not
|
||||
# have decided yet (prod: infra/PROD-PROVISIONING.md §2, whose decision is itself blocked
|
||||
@@ -16,6 +16,12 @@
|
||||
# is: it is the ONE thing that must land on a prod node BEFORE k3s is installed (§0b), and
|
||||
# it must be runnable without touching a disk decision that is still open:
|
||||
# ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags kernel
|
||||
# `wireless` (a cluster node gets exactly ONE path onto the network) is separated for a third
|
||||
# reason: it must be runnable BEFORE `dns` and without touching packages, the kernel or a disk
|
||||
# decision, because a node with a second DHCP-managed interface reverts the `dns` write. It is
|
||||
# placed before the disk work for the same reason, and the pair is the whole remedy for the
|
||||
# k3s-prod-3 incident:
|
||||
# ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags wireless,dns
|
||||
|
||||
- name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs)
|
||||
ansible.builtin.apt:
|
||||
@@ -314,6 +320,304 @@
|
||||
state: "{{ 'started' if node_baseline_qga_channel.stat.exists else 'stopped' }}"
|
||||
tags: [system]
|
||||
|
||||
# =====================================================================================
|
||||
# WIRELESS — tag `wireless`: a cluster node gets exactly ONE path onto the network
|
||||
# =====================================================================================
|
||||
# WHAT WENT WRONG (k3s-prod-3, reported by the operator 2026-09-04). The node was up on BOTH
|
||||
# eno1 (192.168.2.12 — the inventory address) and wlp1s0 (192.168.2.195, DHCP, the SAME /24).
|
||||
# The wifi lease carried a dhcpcd run with it, and dhcpcd's `20-resolv.conf` hook REWROTE
|
||||
# /etc/resolv.conf, undoing the resolvers this role had just written: the file read
|
||||
# "# Generated by dhcpcd from wlp1s0.dhcp, wlp1s0.dhcp6, wlp1s0.ra" with nameserver
|
||||
# 192.168.2.1 plus two Google IPv6 servers, timestamped THREE MINUTES AFTER the role's own
|
||||
# write (01:32 -> 01:35). The role had reported `changed` and green. That is exactly the
|
||||
# "successful run that fixed nothing" the `contended` branch of the DNS block below exists to
|
||||
# prevent, and the detection blind spot that let it through is closed there.
|
||||
#
|
||||
# WHY THE INTERFACE IS REMOVED RATHER THAN JUST REPORTED. Closing the detection hole alone
|
||||
# would have turned a silent revert into a loud failure — an improvement, but it leaves the
|
||||
# node with a second, DHCP-managed path onto the LAN, and on a k3s server that is three
|
||||
# distinct problems beyond DNS:
|
||||
# * a SECOND DEFAULT ROUTE, whose metric decides which source address the node uses;
|
||||
# * a SECOND ADDRESS ON THE SAME L2 — the segment where kube-vip floats 192.168.2.9 and
|
||||
# Cilium L2-announces 192.168.2.20-.30 (§4). One host answering ARP on one subnet from
|
||||
# two NICs is the exact shape neither mechanism can reason about;
|
||||
# * an UNMANAGED address: nothing in this repo knows about .195, and the DHCP server is
|
||||
# free to hand out a different one after any lease expiry.
|
||||
# None of that is a resolver bug, and none of it is fixed by writing resolv.conf harder.
|
||||
#
|
||||
# WHY THE DRIVER LAYER — the four alternatives, and why each is weaker. What has to hold is:
|
||||
# survives a REBOOT, survives a LEASE RENEWAL, and cannot be undone by a routine `ifup`.
|
||||
# Measured on the nodes themselves — all three are HP EliteDesk 800 G5 minis, eno1 is
|
||||
# `e1000e`, the wifi NIC is an Intel AX200 on `iwlwifi`, and the OS is Debian 13 trixie with
|
||||
# ifupdown + `networking.service`, NO NetworkManager, NO systemd-networkd, NO systemd-resolved:
|
||||
# `ip link set <wifi> down` — NOT persistent. Any `ifup`, any hotplug event, any reboot
|
||||
# undoes it. It is what had already been done by hand on node 3, and the stale dhcpcd
|
||||
# /etc/resolv.conf outlived it — the link was down and the wrong resolvers were still
|
||||
# in the file, across a reboot.
|
||||
# mask the service that manages it — THERE IS NO SUCH SERVICE HERE. dhcpcd on these nodes
|
||||
# is `dhcpcd-base`: no dhcpcd.service exists at all (`systemctl is-enabled dhcpcd` =
|
||||
# not-found), it is spawned per-interface by ifupdown. The only unit that can bring
|
||||
# wireless up is `networking.service` — which ALSO owns eno1's static address, so
|
||||
# masking it would cut the one SSH path to a machine whose operator is not on site.
|
||||
# `rfkill` soft-block — weaker on every axis. The netdev still EXISTS, so ifupdown and
|
||||
# dhcpcd can still be pointed at it; persistence depends on systemd-rfkill restoring
|
||||
# saved state (the kernel's own `persistent` flag reads 0 on these devices, and the
|
||||
# `rfkill` CLI is not even installed); and one `rfkill unblock wlan` undoes it.
|
||||
# the BIOS `M.2 WLAN/BT` toggle — genuinely the deepest layer, and it reads `Enable` on all
|
||||
# three nodes (hp-bioscfg survey, §1b). NOT taken here: it needs a REBOOT to take
|
||||
# effect, and a prod reboot is a serialized, supervised operator step (§0b "SECOND
|
||||
# FAULT" — k3s-prod-2 hung on a warm reboot and needed the power button). It also
|
||||
# disables Bluetooth on the same M.2 card. Left as an operator recommendation in §4.
|
||||
# THE DRIVER MODULE — chosen. `blacklist` plus `install <module> /bin/false` in
|
||||
# /etc/modprobe.d means kmod refuses to load the wifi driver on this boot and every
|
||||
# future one, so THE NETDEV NEVER EXISTS. No interface means no DHCP lease, no
|
||||
# resolv.conf hook, no second route, no second address — and every re-activation path
|
||||
# above (udev hotplug, a hand-run `ifup`, wpa_supplicant, a hand-run dhcpcd) needs a
|
||||
# device that is not there. `blacklist` alone would still allow a dependency- or
|
||||
# hand-triggered `modprobe`; the `install ... /bin/false` line is what makes it refuse
|
||||
# outright. It is ONE declarative file, readable on the node, and undone by deleting
|
||||
# it. Paired with the runtime unload below, it needs NO reboot to take effect.
|
||||
#
|
||||
# SAFE AND IDEMPOTENT ON ALL THREE PROD NODES *AND* ON THE LIVE DEV CLUSTER. Nothing here is
|
||||
# hostname- or hardware-specific: the block acts only on interfaces the node itself reports as
|
||||
# real 802.11 devices (a `phy80211` link in sysfs) and on the driver modules behind them.
|
||||
# * dev (Proxmox VMs, virtio NICs) has no such interface, so the discovery comes back empty,
|
||||
# every task below is skipped, and NOT EVEN THE modprobe.d FILE IS WRITTEN. The block
|
||||
# cannot touch dev's networking, which is why it is safe to have on by default.
|
||||
# * k3s-prod-1 (wlp1s0) and k3s-prod-2 (wlp2s0) already have the link DOWN, so the link-down
|
||||
# task is skipped by its own `when` and only the modprobe.d file and the module unload
|
||||
# remain — and both are no-ops on a re-run.
|
||||
# * k3s-prod-3 (wlp1s0) is the reported node.
|
||||
# A re-run changes nothing on any of them: the file is content-identical, `ip link set down`
|
||||
# is gated on the link not already being down, and the unload is gated on the module still
|
||||
# being loaded.
|
||||
#
|
||||
# AND IT REFUSES TO CUT ITS OWN PATH IN. The assert below stops the play if the address
|
||||
# Ansible is connected to is bound to a wireless interface, or if the default route leaves
|
||||
# over one, or if the wireless driver module is also the driver of the default-route
|
||||
# interface. That is the one case where doing this would lock everybody out of a machine
|
||||
# nobody can walk up to, so it is checked per node, on every run, from that node's own facts.
|
||||
- name: Survey this node's 802.11 devices, their drivers and any live DHCP client
|
||||
ansible.builtin.shell:
|
||||
executable: /bin/sh
|
||||
cmd: |
|
||||
set -u
|
||||
# Flat KEY|value lines so the Ansible side needs one single-group regex per fact and
|
||||
# no parsing cleverness. A wireless interface is one with a phy80211 link in sysfs —
|
||||
# that is the kernel's own answer to "is this 802.11", not a name pattern like wlp*.
|
||||
wifi=''
|
||||
for d in /sys/class/net/*; do
|
||||
[ -e "$d/phy80211" ] || continue
|
||||
i=$(basename "$d")
|
||||
wifi="$wifi $i"
|
||||
m=$(basename "$(readlink -f "$d/device/driver" 2>/dev/null || echo unknown)")
|
||||
state=$(cat "$d/operstate" 2>/dev/null || echo unknown)
|
||||
printf 'IFACE|%s\n' "$i"
|
||||
printf 'DETAIL|%s driver=%s operstate=%s\n' "$i" "$m" "$state"
|
||||
# LIVE means "not already down" — the link-down task keys off this, so a node whose
|
||||
# radio is already down reports no change.
|
||||
[ "$state" = down ] || printf 'IFACE_LIVE|%s\n' "$i"
|
||||
ip -o addr show dev "$i" 2>/dev/null | awk '{print "IFACE_ADDR|" $4}'
|
||||
[ "$m" = unknown ] || printf 'MODULE|%s\n' "$m"
|
||||
done
|
||||
for m in $(printf '%s\n' $wifi | while read -r i; do
|
||||
[ -n "$i" ] && basename "$(readlink -f "/sys/class/net/$i/device/driver" 2>/dev/null || echo unknown)"
|
||||
done | sort -u); do
|
||||
[ "$m" = unknown ] && continue
|
||||
[ -d "/sys/module/$m" ] || continue
|
||||
printf 'MODULE_LOADED|%s\n' "$m"
|
||||
# Holders must be unloaded FIRST: `modprobe -r iwlwifi` fails while iwlmvm still
|
||||
# holds it. Read from sysfs rather than assumed, so a different wifi driver stack
|
||||
# (ath*, rtw*, mt76*) works without editing this role.
|
||||
for h in $(ls "/sys/module/$m/holders" 2>/dev/null); do printf 'MODULE_HOLDER|%s\n' "$h"; done
|
||||
done
|
||||
dr=$(ip -o route show default 2>/dev/null | awk '{for (i = 1; i < NF; i++) if ($i == "dev") print $(i + 1)}' | head -1)
|
||||
printf 'DEFROUTE_IFACE|%s\n' "${dr:-none}"
|
||||
if [ -n "${dr:-}" ] && [ -e "/sys/class/net/$dr" ]; then
|
||||
printf 'DEFROUTE_MODULE|%s\n' "$(basename "$(readlink -f "/sys/class/net/$dr/device/driver" 2>/dev/null || echo unknown)")"
|
||||
fi
|
||||
# Any DHCP client process, with the interfaces it was started for. Used only to stop a
|
||||
# client that holds a WIRELESS interface and nothing else.
|
||||
for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do
|
||||
printf 'DHCPCLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)"
|
||||
done
|
||||
exit 0
|
||||
register: node_baseline_wifi_scan
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Interpret the wireless survey
|
||||
ansible.builtin.set_fact:
|
||||
node_baseline_wifi_ifaces: "{{ scan | regex_findall('(?m)^IFACE\\|(.+)$') }}"
|
||||
node_baseline_wifi_live: "{{ scan | regex_findall('(?m)^IFACE_LIVE\\|(.+)$') }}"
|
||||
node_baseline_wifi_addrs: "{{ scan | regex_findall('(?m)^IFACE_ADDR\\|(\\S+)') | map('regex_replace', '/.*$', '') | list }}"
|
||||
node_baseline_wifi_modules: "{{ scan | regex_findall('(?m)^MODULE\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_modules_loaded: "{{ scan | regex_findall('(?m)^MODULE_LOADED\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_module_holders: "{{ scan | regex_findall('(?m)^MODULE_HOLDER\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_detail: "{{ scan | regex_findall('(?m)^DETAIL\\|(.+)$') }}"
|
||||
node_baseline_wifi_dhcp_clients: "{{ scan | regex_findall('(?m)^DHCPCLIENT\\|(.+)$') }}"
|
||||
node_baseline_defroute_iface: "{{ (scan | regex_findall('(?m)^DEFROUTE_IFACE\\|(.+)$') | first) | default('none', true) }}"
|
||||
node_baseline_defroute_module: "{{ (scan | regex_findall('(?m)^DEFROUTE_MODULE\\|(.+)$') | first) | default('unknown', true) }}"
|
||||
vars:
|
||||
scan: "{{ node_baseline_wifi_scan.stdout | default('') }}"
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Report this node's wireless hardware
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
wireless devices = {{ node_baseline_wifi_detail | default([]) | join('; ') | default('(none)', true) }}
|
||||
| driver modules = {{ node_baseline_wifi_modules | default([]) | join(', ') | default('(none)', true) }}
|
||||
(loaded: {{ node_baseline_wifi_modules_loaded | default([]) | join(', ') | default('(none)', true) }},
|
||||
held by: {{ node_baseline_wifi_module_holders | default([]) | join(', ') | default('(none)', true) }})
|
||||
| default route via {{ node_baseline_defroute_iface | default('?') }}
|
||||
(driver {{ node_baseline_defroute_module | default('?') }})
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# The one way this block could do real harm is by removing the interface it is being run
|
||||
# over. Checked from the node's own facts, per node, every run — not assumed from the fact
|
||||
# that prod "should" be wired.
|
||||
- name: Refuse to disable wireless if the path to this node runs OVER the wireless interface
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- node_baseline_defroute_iface not in node_baseline_wifi_ifaces
|
||||
- node_baseline_defroute_module not in node_baseline_wifi_modules
|
||||
- (ansible_host | default(inventory_hostname)) not in node_baseline_wifi_addrs
|
||||
fail_msg: >-
|
||||
REFUSING to disable wireless on this node: the connection to it appears to BE the
|
||||
wireless path. ansible_host={{ ansible_host | default(inventory_hostname) }}, wireless
|
||||
interfaces={{ node_baseline_wifi_ifaces | join(', ') }} carrying
|
||||
{{ node_baseline_wifi_addrs | join(', ') | default('no addresses', true) }}, default route
|
||||
via {{ node_baseline_defroute_iface }} (driver {{ node_baseline_defroute_module }}).
|
||||
Disabling the driver here would drop this node off the network with no remote way back.
|
||||
Give the node a wired address, point the inventory at it, and re-run.
|
||||
quiet: true
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_ifaces | length > 0
|
||||
tags: [wireless]
|
||||
|
||||
# THE PERSISTENT HALF. This file is the whole reboot story: kmod consults /etc/modprobe.d on
|
||||
# every load attempt, so from here on the driver cannot be loaded by udev at boot, as a
|
||||
# dependency, or by hand. Written ONLY when the node actually has a wireless driver, so a
|
||||
# node without wifi hardware (every dev VM) gets no file and no change at all.
|
||||
# `update-initramfs` is deliberately NOT run: no wifi module is loaded from the initramfs on
|
||||
# these nodes (root is a local SATA/NVMe device), the driver is autoloaded by udev after
|
||||
# switch-root, and re-generating an initramfs on a prod node is a bigger risk than the one it
|
||||
# would cover.
|
||||
- name: Block the wireless driver module(s) from ever loading again
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ node_baseline_wireless_modprobe_file }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
# Managed by the Ansible node_baseline role (tags: wireless) — do not edit by hand.
|
||||
#
|
||||
# WHY THIS NODE HAS NO WIRELESS. A k3s server gets exactly ONE path onto the network.
|
||||
# A second, DHCP-managed interface on the same LAN gives the node a second default
|
||||
# route, a second address nothing in the cluster's config knows about, and a DHCP
|
||||
# client whose resolv.conf hook silently overwrites the resolvers Ansible manages
|
||||
# (that is what happened on k3s-prod-3, 2026-09-04). The link is also brought down and
|
||||
# the module unloaded at the same time, so no reboot is needed for this to take effect.
|
||||
#
|
||||
# `blacklist` stops udev's alias-driven autoload; the `install ... /bin/false` line is
|
||||
# what also refuses a dependency-triggered or hand-run `modprobe`. Both are needed.
|
||||
#
|
||||
# TO RE-ENABLE WIRELESS ON THIS NODE: delete this file and `modprobe` the module(s)
|
||||
# below (or reboot). To stop Ansible putting it back, set
|
||||
# node_baseline_disable_wireless: false for this host first — otherwise the next
|
||||
# node-baseline run recreates it. Full rationale: roles/node_baseline/tasks/main.yml,
|
||||
# the `wireless` block, and infra/PROD-PROVISIONING.md §4.
|
||||
{% for module in node_baseline_wifi_modules %}
|
||||
blacklist {{ module }}
|
||||
install {{ module }} /bin/false
|
||||
{% endfor %}
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_modules | length > 0
|
||||
tags: [wireless]
|
||||
|
||||
# THE RUNTIME HALF — everything below makes the change true NOW, so the node does not have to
|
||||
# be rebooted to stop carrying a second address (this role never reboots: §0b "SECOND FAULT").
|
||||
#
|
||||
# Stop a DHCP client that is holding a wireless interface first, so it releases its lease
|
||||
# instead of being left holding a device that is about to disappear. HEAVILY GUARDED: it runs
|
||||
# only for clients whose command line names a wireless interface and names NO non-wireless
|
||||
# interface, because `dhcpcd -x` against a shared master process would also drop a wired
|
||||
# lease. On these prod nodes eno1 is a static ifupdown address with no DHCP client at all, so
|
||||
# this task has nothing to do; it exists so the ordering is correct on a node where wifi is
|
||||
# still actively leasing.
|
||||
- name: Stop the dhcpcd instance that holds a wireless interface (and nothing else)
|
||||
ansible.builtin.command:
|
||||
argv: [dhcpcd, --exit, "{{ item }}"]
|
||||
loop: "{{ node_baseline_wifi_ifaces }}"
|
||||
register: node_baseline_wifi_dhcpcd_exit
|
||||
changed_when: rc == 0
|
||||
failed_when: false
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_dhcp_wifi_only | length > 0
|
||||
- node_baseline_wifi_dhcp_wifi_only | select('search', 'dhcpcd') | list | length > 0
|
||||
vars:
|
||||
# A client is "wireless-only" when its argv mentions a wireless interface and mentions no
|
||||
# other interface this node has. Empty interface sets are turned into a pattern that
|
||||
# cannot match, so an odd node fails towards doing nothing rather than towards guessing.
|
||||
node_baseline_non_wifi_ifaces: "{{ ansible_facts['interfaces'] | default([]) | difference(node_baseline_wifi_ifaces) | difference(['lo']) }}"
|
||||
node_baseline_wifi_dhcp_wifi_only: >-
|
||||
{{ node_baseline_wifi_dhcp_clients
|
||||
| select('search', '\b(' ~ ((node_baseline_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b')
|
||||
| reject('search', '\b(' ~ ((node_baseline_non_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b')
|
||||
| list }}
|
||||
tags: [wireless]
|
||||
|
||||
- name: Take each wireless link down now
|
||||
ansible.builtin.command:
|
||||
argv: [ip, link, set, dev, "{{ item }}", down]
|
||||
loop: "{{ node_baseline_wifi_live }}"
|
||||
changed_when: true
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# Unload the driver so the netdev is gone immediately — holders (iwlmvm on these nodes) first,
|
||||
# then the driver itself, because `modprobe -r` refuses a module that is still held.
|
||||
# NOT fatal on failure: the persistent block above is already in place, so a module that
|
||||
# cannot be unloaded right now is gone at the next boot regardless. The report below says so
|
||||
# out loud rather than letting a green play imply the radio is already off.
|
||||
- name: Unload the wireless driver module(s) now (no reboot needed)
|
||||
community.general.modprobe:
|
||||
name: "{{ item }}"
|
||||
state: absent
|
||||
loop: "{{ node_baseline_wifi_module_holders + node_baseline_wifi_modules_loaded }}"
|
||||
register: node_baseline_wifi_unload
|
||||
failed_when: false
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Report the wireless state this node is left in
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ 'No 802.11 device on this node — nothing to disable (no modprobe.d file written).'
|
||||
if (node_baseline_wifi_ifaces | length) == 0
|
||||
else ((('WIRELESS DISABLED: ' ~ (node_baseline_wifi_modules | join(', ')) ~ ' blocked in '
|
||||
~ node_baseline_wireless_modprobe_file ~ ' (survives reboot) and unloaded now; '
|
||||
~ (node_baseline_wifi_ifaces | join(', ')) ~ ' is down and the netdev is gone.')
|
||||
if (failed_unloads | length) == 0
|
||||
else ('WIRELESS BLOCKED FOR THE NEXT BOOT BUT STILL LOADED NOW: could not unload '
|
||||
~ (failed_unloads | join(', ')) ~ '. ' ~ node_baseline_wireless_modprobe_file
|
||||
~ ' is in place, so the driver will not load again after a reboot, but the '
|
||||
~ 'radio is live until then — check `ip link` and whether something still '
|
||||
~ 'holds the device.'))) }}
|
||||
vars:
|
||||
failed_unloads: >-
|
||||
{{ (node_baseline_wifi_unload.results | default([]))
|
||||
| selectattr('failed', 'defined') | selectattr('failed') | map(attribute='item') | list }}
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# =====================================================================================
|
||||
# DATA DISKS — tag `disks`, in tasks/disks.yml
|
||||
# =====================================================================================
|
||||
@@ -377,6 +681,11 @@
|
||||
# node and the matching seam is written. Adding a third mechanism means adding a branch
|
||||
# here, not rewriting the role.
|
||||
#
|
||||
# AND DETECTION IS ONLY AS GOOD AS THE SET OF OWNERS IT KNOWS ABOUT. It originally looked for
|
||||
# resolvconf/openresolv and NetworkManager, missed dhcpcd, and therefore reported a green,
|
||||
# `changed` run on k3s-prod-3 whose resolvers a wifi lease reverted three minutes later. dhcpcd
|
||||
# is now part of that set — see the detection task below and the `contended` failure.
|
||||
#
|
||||
# All of it is `dns`-tagged, so `--tags packages` (the prod package-floor escape hatch
|
||||
# documented at the top of this file) still installs nothing but packages.
|
||||
|
||||
@@ -396,6 +705,55 @@
|
||||
check_mode: false
|
||||
tags: [dns]
|
||||
|
||||
# THE dhcpcd BLIND SPOT, CLOSED (k3s-prod-3, 2026-09-04). dhcpcd's `20-resolv.conf` hook
|
||||
# rewrites /etc/resolv.conf on every lease, renewal and carrier change, and it is a THIRD
|
||||
# way to own that file — neither resolvconf nor NetworkManager. On node 3 it silently reverted
|
||||
# this role's write three minutes later, and because nothing below looked for it the role had
|
||||
# already reported `changed` and green: the precise failure the `contended` branch exists to
|
||||
# make impossible.
|
||||
#
|
||||
# WHY THIS LOOKS FOR A LIVE CLIENT AND NOT FOR AN INSTALLED PACKAGE. `dhcpcd-base` is
|
||||
# installed on all three prod nodes and always will be — it is the DHCP client ifupdown
|
||||
# reaches for — and there is NO dhcpcd.service on them at all (`systemctl is-active dhcpcd` =
|
||||
# not-found; it is spawned per-interface by ifupdown). Keying off the package, or off a unit
|
||||
# that does not exist, would put every prod node permanently in `contended` and leave the DNS
|
||||
# block unable to configure a node it is perfectly safe to configure. What actually matters is
|
||||
# whether a DHCP client CAN still write the file, so all three real signals are checked:
|
||||
# CLIENT| a dhcpcd/dhclient process running right now (with the argv that started it);
|
||||
# UNIT| a dhcpcd/dhclient unit that IS active or IS enabled, on a distro that has one;
|
||||
# STANZA| an UNCOMMENTED `iface <x> inet[6] dhcp` in /etc/network/interfaces{,.d} — the
|
||||
# configuration that makes ifupdown spawn a client at the next boot or `ifup`, even
|
||||
# when none is running at this instant.
|
||||
# Any one of them means "another manager is in play" and the play stops below. None of them
|
||||
# means no client can run, and the file is genuinely ours to own.
|
||||
- name: Determine whether a DHCP client (dhcpcd/dhclient) can rewrite /etc/resolv.conf
|
||||
ansible.builtin.shell:
|
||||
executable: /bin/sh
|
||||
cmd: |
|
||||
set -u
|
||||
for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do
|
||||
printf 'CLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)"
|
||||
done
|
||||
for u in dhcpcd dhcpcd5 dhclient isc-dhcp-client; do
|
||||
a=$(systemctl is-active "$u" 2>/dev/null || true)
|
||||
e=$(systemctl is-enabled "$u" 2>/dev/null || true)
|
||||
# Compared exactly, never with a substring match: `is-active` answers "inactive",
|
||||
# which contains the word "active".
|
||||
if [ "$a" = active ] || [ "$e" = enabled ] || [ "$e" = enabled-runtime ]; then
|
||||
printf 'UNIT|%s active=%s enabled=%s\n' "$u" "$a" "$e"
|
||||
fi
|
||||
done
|
||||
# `^[[:space:]]*iface` cannot match a commented-out stanza, which is how the wireless
|
||||
# stanza is left on these nodes.
|
||||
grep -rhE '^[[:space:]]*iface[[:space:]]+[^[:space:]]+[[:space:]]+inet6?[[:space:]]+dhcp' \
|
||||
/etc/network/interfaces /etc/network/interfaces.d/ 2>/dev/null \
|
||||
| sed -e 's/^[[:space:]]*/STANZA|/'
|
||||
exit 0
|
||||
register: node_baseline_dhcp_client_scan
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
tags: [dns]
|
||||
|
||||
# follow: false — we need to know whether /etc/resolv.conf IS a symlink and where it
|
||||
# points, which is the strongest single signal of who owns the file.
|
||||
- name: Inspect /etc/resolv.conf itself (symlink or real file, and its target)
|
||||
@@ -426,9 +784,10 @@
|
||||
# resolved's loopback stub (127.0.0.53 / 127.0.0.54). Writing a resolved drop-in is
|
||||
# then the ONLY thing that changes real behaviour; writing resolv.conf would be
|
||||
# overwritten by resolved (or, worse, land in /run and vanish at reboot).
|
||||
# contended — nobody's stub, but resolvconf or NetworkManager is present/running and
|
||||
# would regenerate the file underneath us. We do NOT write. The role fails loudly
|
||||
# below rather than leaving an operator believing a change stuck when it did not.
|
||||
# contended — nobody's stub, but resolvconf, NetworkManager or a DHCP client (dhcpcd /
|
||||
# dhclient) is present/running/configured and would regenerate the file underneath us.
|
||||
# We do NOT write. The role fails loudly below rather than leaving an operator
|
||||
# believing a change stuck when it did not.
|
||||
# resolv-conf-file — a plain, directly-managed /etc/resolv.conf (prod's shape today).
|
||||
# Safe to own outright.
|
||||
# resolved running but NOT owning resolv.conf correctly falls through to the file branch:
|
||||
@@ -438,13 +797,21 @@
|
||||
node_baseline_dns_mechanism: >-
|
||||
{{
|
||||
'systemd-resolved' if (node_baseline_resolved_running and node_baseline_resolved_owns_resolv)
|
||||
else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present)
|
||||
else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present
|
||||
or node_baseline_dhcp_client_live)
|
||||
else 'resolv-conf-file')
|
||||
}}
|
||||
# Kept as facts (not just task vars) so the failure message, the report and the
|
||||
# reclaim note below all quote the SAME evidence this decision was made on.
|
||||
node_baseline_dhcp_client_evidence: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) | regex_findall('(?m)^(?:CLIENT|UNIT|STANZA)\\|(.+)$') }}"
|
||||
# A file dhcpcd WROTE, versus a dhcpcd that can still write it — two different facts, and
|
||||
# only the second one is contention. See the reclaim note below.
|
||||
node_baseline_resolv_dhcpcd_written: "{{ 'Generated by dhcpcd' in (node_baseline_resolv_slurp.content | default('') | b64decode) }}"
|
||||
vars:
|
||||
node_baseline_resolved_running: "{{ (node_baseline_resolved_active.stdout | default('') | trim) == 'active' }}"
|
||||
node_baseline_nm_running: "{{ (node_baseline_nm_active.stdout | default('') | trim) == 'active' }}"
|
||||
node_baseline_resolvconf_present: "{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}"
|
||||
node_baseline_dhcp_client_live: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) is search('(?m)^(?:CLIENT|UNIT|STANZA)\\|') }}"
|
||||
node_baseline_resolv_text: "{{ (node_baseline_resolv_slurp.content | default('') | b64decode) }}"
|
||||
node_baseline_resolv_ns: "{{ node_baseline_resolv_text | regex_findall('(?m)^\\s*nameserver\\s+(\\S+)') }}"
|
||||
node_baseline_resolved_owns_resolv: >-
|
||||
@@ -461,7 +828,9 @@
|
||||
NetworkManager is-active={{ node_baseline_nm_active.stdout | default('') | trim }},
|
||||
/etc/resolv.conf islnk={{ node_baseline_resolv_stat.stat.islnk | default(false) }}
|
||||
target={{ node_baseline_resolv_stat.stat.lnk_source | default('(not a symlink)') }},
|
||||
resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }})
|
||||
resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }},
|
||||
DHCP client evidence={{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }},
|
||||
resolv.conf written by dhcpcd={{ node_baseline_resolv_dhcpcd_written | default(false) }})
|
||||
tags: [dns]
|
||||
|
||||
# Same shape, and the same reason, as the data-disk assert above: this environment's
|
||||
@@ -494,16 +863,45 @@
|
||||
- name: Refuse to manage /etc/resolv.conf while another resolver manager owns it
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
/etc/resolv.conf on this node is not the systemd-resolved stub, but
|
||||
resolvconf/openresolv (/etc/resolvconf/resolv.conf.d present:
|
||||
{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) or NetworkManager
|
||||
(is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}) is in play and
|
||||
would regenerate the file underneath this role. Writing it here would look like it
|
||||
worked and silently revert. Add the matching branch to roles/node_baseline (the
|
||||
`dns`-tagged block) for whichever manager this node really uses — do not force it.
|
||||
/etc/resolv.conf on this node is not the systemd-resolved stub, but something else would
|
||||
regenerate the file underneath this role: resolvconf/openresolv
|
||||
(/etc/resolvconf/resolv.conf.d present:
|
||||
{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}), NetworkManager
|
||||
(is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}), or a DHCP client
|
||||
that can still write it — dhcpcd/dhclient evidence:
|
||||
{{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}.
|
||||
Writing it here would look like it worked and silently revert — which is exactly what
|
||||
happened on k3s-prod-3 on 2026-09-04, where a wifi lease's dhcpcd hook reverted this
|
||||
role's resolvers three minutes after a green run. If the evidence above is a DHCP client
|
||||
on a WIRELESS interface, the fix is the `wireless` block earlier in this role (a cluster
|
||||
node gets one network path) — run it, then re-run this. If it is a DHCP client on the
|
||||
node's real interface, or another manager entirely, add the matching branch to
|
||||
roles/node_baseline for the manager this node really uses — do not force it.
|
||||
when: node_baseline_dns_mechanism == 'contended'
|
||||
tags: [dns]
|
||||
|
||||
# THE CORPSE CASE, and why it is NOT a hole in the fail-closed rule above. After the
|
||||
# `wireless` block disables the radio, k3s-prod-3 is left with a /etc/resolv.conf that dhcpcd
|
||||
# WROTE (its "# Generated by dhcpcd from wlp1s0.dhcp..." header, the gateway as sole IPv4
|
||||
# resolver, two Google IPv6 servers) and NO dhcpcd that can ever run again. `contended` means
|
||||
# "another manager is live and will undo this write"; a file left behind by a client that no
|
||||
# longer exists is not a manager, it is the damage — and refusing to write here would leave the
|
||||
# node permanently mis-resolved, which is the very outcome the operator reported. So the file
|
||||
# branch below reclaims it, and says so.
|
||||
- name: Note that a dhcpcd-written /etc/resolv.conf is being reclaimed
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
/etc/resolv.conf on this node was written by dhcpcd, but no DHCP client can write it any
|
||||
more (no process, no active/enabled unit, no `inet dhcp` stanza). Taking ownership and
|
||||
replacing it with the declared upstreams
|
||||
({{ node_baseline_dns_servers | join(', ') }}). If a DHCP client ever comes back on this
|
||||
node, the `contended` check above will stop the play instead of letting it revert
|
||||
silently.
|
||||
when:
|
||||
- node_baseline_dns_mechanism == 'resolv-conf-file'
|
||||
- node_baseline_resolv_dhcpcd_written | default(false)
|
||||
tags: [dns]
|
||||
|
||||
# --- Mechanism A: systemd-resolved (dev's shape today) -------------------------------
|
||||
# A drop-in, never an edit of /etc/systemd/resolved.conf, so a package upgrade of that file
|
||||
# never conflicts. `DNS=` is one ORDERED list inside a single resolved scope: resolved uses
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
# platform-config-cdi — the SECOND-ORDER config wave: custom resources whose CRD is created at
|
||||
# RUNTIME by a controller that a `platform-config` custom resource switches on.
|
||||
#
|
||||
# ── THE BUG THIS DIRECTORY EXISTS TO FIX (observed on PROD, 2026-09-05) ───────────────────────────
|
||||
# The repo already splits CONTROLLERS (`platform`) from CONFIGS (`platform-config`) because a CR
|
||||
# cannot be dry-run before its CRD is registered. That split assumes a CRD is registered by the
|
||||
# controller INSTALL. CDI breaks the assumption: `cdi-operator.yaml` registers exactly ONE CRD
|
||||
# (`cdis.cdi.kubevirt.io`). Every OTHER CDI CRD — `storageprofiles`, `datavolumes`, `datasources`,
|
||||
# `dataimportcrons`, … — is created by cdi-operator only AFTER it has reconciled the `CDI` CR.
|
||||
#
|
||||
# So `cdi-cr.yaml` is a FIRST-order CR (its CRD comes from the install) and belongs in
|
||||
# platform-config, while `cdi-storageprofiles.yaml` is a SECOND-order CR (its CRD comes from
|
||||
# reconciling the first-order CR). Having both in ONE Flux Kustomization is a true DEADLOCK, not a
|
||||
# race: kustomize-controller server-side dry-runs EVERY object before applying ANY of them, so the
|
||||
# unknown `StorageProfile` kind aborts the whole pass → the `CDI` CR is never applied → the
|
||||
# StorageProfile CRD is never created → platform-config can NEVER converge, and every tier that
|
||||
# dependsOn it stays blocked forever. On prod, cdi-operator sat Available for 12h with no CR:
|
||||
#
|
||||
# StorageProfile/longhorn dry-run failed: no matches for kind "StorageProfile"
|
||||
# in version "cdi.kubevirt.io/v1beta1"
|
||||
#
|
||||
# Invisible on dev (whose CDI CRDs have existed for months) and fatal on any COLD cluster.
|
||||
#
|
||||
# ── THE RULE, so this is not rediscovered a third time ────────────────────────────────────────────
|
||||
# A CR belongs HERE (not in platform-config) when the CRD it needs is created by a CONTROLLER AT
|
||||
# RUNTIME rather than by a manifest in git. Grep test: if `grep -rn "kind: CustomResourceDefinition"`
|
||||
# over ./platform (following remote bases) does not produce the CRD, and no chart installs it, then
|
||||
# some operator creates it after being switched on — and this is the tier for it.
|
||||
#
|
||||
# Files are referenced IN PLACE via ../platform/... (Flux builds with LoadRestrictionsNone, and the
|
||||
# manifests stay beside the component they configure). Build/verify locally with:
|
||||
# kubectl kustomize --load-restrictor LoadRestrictionsNone platform-config-cdi
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
# CDI StorageProfiles — snapshot smart-clone for `longhorn`, and RWX/Block + snapshot clone for the
|
||||
# `longhorn-vm` virt-default class (clears CDIDefaultStorageClassDegraded). `StorageProfile`'s CRD
|
||||
# is created by cdi-operator only once the `CDI` CR (applied by platform-config) is reconciled,
|
||||
# which is exactly why these cannot live in the same pass as that CR.
|
||||
- ../platform/kubevirt/cdi-storageprofiles.yaml
|
||||
@@ -25,6 +25,14 @@ resources:
|
||||
- ../platform/kyverno/policies/restrict-host-namespaces.yaml
|
||||
- ../platform/kyverno/policies/require-egress-class-label.yaml
|
||||
- ../platform/kyverno/policies/require-auth-mode-label.yaml
|
||||
# COLD-START FIX (2026-09-06, prod): a MUTATE policy that must travel WITH the two label
|
||||
# policies above. CDI generates its own short-lived worker pods (importer-*, uploader-*,
|
||||
# cloner-*) inside app namespaces; nothing in git can label them, so both policies DENIED them
|
||||
# and `DataVolume unifi/unifi-rootdisk` sat in ImportScheduled with no disk. This policy adds
|
||||
# the two labels to pods created by ServiceAccount cdi/cdi-sa, so the requirement is SATISFIED
|
||||
# rather than bypassed. Applying it here (not later) matters: if it lands after the validating
|
||||
# policies, the next import is blocked in the gap.
|
||||
- ../platform/kyverno/policies/label-cdi-worker-pods.yaml
|
||||
- ../platform/kyverno/admission/secret-provenance.yaml
|
||||
# cilium: L2 LB IP pool + announcement policy (cilium.io CRDs already registered by Cilium)
|
||||
- cilium/l2-ip-pool.yaml
|
||||
@@ -41,9 +49,23 @@ resources:
|
||||
# ConfigMap kept beside them for locality.
|
||||
- ../platform/kubevirt/kubevirt-cr.yaml
|
||||
- ../platform/kubevirt/cdi-cr.yaml
|
||||
# CDI StorageProfiles: snapshot smart-clone for `longhorn`, and RWX/Block + snapshot clone for the
|
||||
# `longhorn-vm` virt-default class (consume cdi.kubevirt.io; clears CDIDefaultStorageClassDegraded).
|
||||
- ../platform/kubevirt/cdi-storageprofiles.yaml
|
||||
# *** DO NOT ADD ../platform/kubevirt/cdi-storageprofiles.yaml BACK HERE. IT DEADLOCKS THIS TIER. ***
|
||||
# The CDI StorageProfiles were MOVED OUT on 2026-09-05 into their own Flux Kustomization
|
||||
# (clusters/base/platform-config-cdi.yaml -> ./platform-config-cdi, dependsOn platform-config).
|
||||
#
|
||||
# `StorageProfile`'s CRD is NOT registered by the CDI operator INSTALL — cdi-operator.yaml ships only
|
||||
# `cdis.cdi.kubevirt.io`. cdi-operator creates `storageprofiles` (and datavolumes, datasources, …)
|
||||
# only AFTER it has reconciled the `CDI` CR two lines above. Flux server-side DRY-RUNS EVERY object
|
||||
# in a Kustomization before applying ANY of them, so having both in this list meant: unknown kind
|
||||
# `StorageProfile` -> the whole pass aborts -> the `CDI` CR is never applied -> the CRD is never
|
||||
# created -> THIS TIER CAN NEVER CONVERGE, and every tier that dependsOn it is blocked forever.
|
||||
# A true deadlock, not a race. Observed on prod 2026-09-05 (cdi-operator Available 12h, no CR):
|
||||
# StorageProfile/longhorn dry-run failed: no matches for kind "StorageProfile" in version
|
||||
# "cdi.kubevirt.io/v1beta1"
|
||||
# Invisible on dev, whose CDI CRDs have existed for months. See DECISIONS-LOG 2026-09-05.
|
||||
#
|
||||
# THE RULE: a CR whose CRD is created by a controller AT RUNTIME (rather than by a manifest in git)
|
||||
# is SECOND-ORDER and belongs in ./platform-config-cdi, not here.
|
||||
- ../platform/kubevirt/servicemonitor.yaml
|
||||
- ../platform/kubevirt/alerts.yaml
|
||||
- ../platform/kubevirt/dashboard.yaml
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
#
|
||||
# SOURCE OF TRUTH (do not diverge without updating both):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version, k3s_api_host/port)
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version)
|
||||
# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint, k3s_api_port)
|
||||
#
|
||||
# Every value tagged [parity] below is asserted identical to the Ansible render. If you change one
|
||||
# here, change it in the Ansible role in the same PR. (docs/02 Cilium-ownership contract.)
|
||||
@@ -45,11 +46,35 @@ spec:
|
||||
# [parity] MUST equal Ansible: kubeProxyReplacement: true
|
||||
kubeProxyReplacement: true
|
||||
# [parity] MUST equal Ansible: k8sServiceHost/k8sServicePort = the k3s API endpoint
|
||||
# (k3s_api_host/k3s_api_port). This is the primary control-plane node IP.
|
||||
# (**k3s_apiserver_endpoint** / k3s_api_port, infra/ansible/inventory/group_vars/all.yml).
|
||||
# Setting these wrong breaks Cilium's connection to the API server on adopt.
|
||||
# ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.50 | prod set in clusters/prod.
|
||||
# The `platform` Flux Kustomization substitutes it; the Ansible side is already parameterized
|
||||
# from inventory (k3s_api_host), so the parity contract holds per-environment, not per-literal.
|
||||
#
|
||||
# THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS — operator decision 2026-09-06.
|
||||
# ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.49 | prod 192.168.2.9, each equal to
|
||||
# that environment's kube_vip_address. The knob's NAME still says "host"; read it as "the API
|
||||
# host Cilium dials", and see clusters/base/cluster-vars.yaml for why it was not renamed.
|
||||
#
|
||||
# WHY, in one line: this value used to be the PRIMARY NODE's IP, so on a 3-server HA cluster the
|
||||
# CNI's control-plane path depended on one machine. Verified on live prod 2026-09-06 — every
|
||||
# cilium-agent carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1) while kube-vip held .9.
|
||||
# Losing that node does not break the datapath (Cilium keeps serving from cache); it silently
|
||||
# stops every service and policy update from ever landing, which is worse to detect.
|
||||
#
|
||||
# [parity] IS NOW AGAINST k3s_apiserver_endpoint AND NOT k3s_api_host. Those were one Ansible
|
||||
# variable until 2026-09-06 and the conflation is the defect. k3s_api_host still exists and
|
||||
# still holds the FOUNDER's own address, because roles/k3s_server's secondary
|
||||
# `--server https://<host>:6443` join must target a host that is already up — on a COLD build
|
||||
# no VIP exists yet. So: founder for the join, VIP for the CNI. If you change this line, change
|
||||
# cilium-values.yaml.j2 in the same PR, and point it at the SAME variable.
|
||||
#
|
||||
# The `platform` Flux Kustomization substitutes it; the Ansible side is parameterized from
|
||||
# inventory (k3s_apiserver_endpoint derives from kube_vip_address), so the parity contract holds
|
||||
# per-environment, not per-literal.
|
||||
#
|
||||
# NEEDS NO CERT WORK: k3s_tls_sans has always carried kube_vip_address alongside every node IP,
|
||||
# so both addresses are already SANs on the serving cert (verified live 2026-09-06 — prod
|
||||
# answers through https://192.168.2.9:6443 under full CA verification, no insecure skip). This
|
||||
# switch reissues nothing and restarts no apiserver.
|
||||
k8sServiceHost: "${K3S_API_HOST}"
|
||||
k8sServicePort: 6443
|
||||
# --- Gateway API (ingress; replaces Traefik) — docs/09, ADR-0002 ---
|
||||
|
||||
@@ -6,16 +6,20 @@
|
||||
#
|
||||
# SOURCE OF TRUTH (all [parity] fields MUST equal these):
|
||||
# infra/ansible/roles/cilium/templates/cilium-values.yaml.j2
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1, k3s_api_host/port)
|
||||
# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1)
|
||||
# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint = the kube-vip VIP,
|
||||
# k3s_api_port) — NOT k3s_api_host, which is the
|
||||
# founder's own address and bootstrap-only
|
||||
#
|
||||
# chart version (helmrelease.yaml chart.spec.version): 1.20.1 # [parity] = cilium_version
|
||||
|
||||
# [parity] k3s kube-proxy disabled → Cilium replaces it (ADR-0001/0002)
|
||||
kubeProxyReplacement: true
|
||||
# [parity] = the k3s API endpoint (primary control-plane node IP) + port.
|
||||
# [parity] = the k3s API endpoint + port. THE CONTROL-PLANE VIP (kube-vip), not the primary node —
|
||||
# changed 2026-09-06; the Ansible half is k3s_apiserver_endpoint, NOT k3s_api_host.
|
||||
# Mirrors the HelmRelease's ${K3S_API_HOST} cluster-vars token (R-ENV-02). This file is NOT a
|
||||
# kustomize resource, so nothing substitutes it — the token is shown with its dev value beside it.
|
||||
k8sServiceHost: "${K3S_API_HOST}" # dev: 192.168.1.50
|
||||
k8sServiceHost: "${K3S_API_HOST}" # the CONTROL-PLANE VIP — dev: 192.168.1.49 | prod: 192.168.2.9
|
||||
k8sServicePort: 6443
|
||||
# [parity] ingress via Gateway API, no Traefik (docs/09)
|
||||
gatewayAPI:
|
||||
|
||||
@@ -29,9 +29,9 @@ metadata:
|
||||
data:
|
||||
# Filename `home.server` -> imported as a standalone server block for the whole zone.
|
||||
home.server: |
|
||||
home.keithjackson.dev:53 {
|
||||
${BASE_DOMAIN}:53 {
|
||||
errors
|
||||
# Answer EVERY name in the zone (apex + all `<svc>.home.keithjackson.dev`) with the Gateway
|
||||
# Answer EVERY name in the zone (apex + all `<svc>.${BASE_DOMAIN}`) with the Gateway
|
||||
# VIP. The template plugin's default match is `.*`, so this covers current and FUTURE app
|
||||
# hostnames without editing this file per service. `{{ .Name }}` echoes the queried FQDN.
|
||||
template IN A {
|
||||
|
||||
@@ -125,7 +125,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
# Gate the co-located Crafty UI. Crafty serves HTTPS with a SELF-SIGNED cert, so skip verify
|
||||
# on this in-pod (127.0.0.1) hop; the browser-facing TLS is the trusted wildcard at the Gateway.
|
||||
- --upstream=https://127.0.0.1:8443
|
||||
@@ -138,9 +138,9 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
# Single provider => 302 straight to Keycloak, no oauth2-proxy interstitial button.
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://crafty.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://crafty.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- crafty.home.keithjackson.dev
|
||||
- crafty.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -42,7 +42,7 @@ metadata:
|
||||
lbipam.cilium.io/sharing-key: "crafty"
|
||||
lbipam.cilium.io/ips: "${CRAFTY_L4_VIP}"
|
||||
# external-dns publishes A crafty-mc.home.keithjackson.dev -> the pinned VIP (Java server target).
|
||||
external-dns.alpha.kubernetes.io/hostname: "crafty-mc.home.keithjackson.dev"
|
||||
external-dns.alpha.kubernetes.io/hostname: "crafty-mc.${BASE_DOMAIN}"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -20,8 +20,12 @@ Neither k3s nor the Longhorn chart ships this. It is a prerequisite for:
|
||||
## Wiring
|
||||
|
||||
- **Pass:** `platform` (controllers). Registers the CRDs.
|
||||
- The `VolumeSnapshotClass` / `StorageProfile` CRs that consume the CRDs live in the
|
||||
`platform-config` pass (`dependsOn: platform`), per the controllers/configs split.
|
||||
- The `VolumeSnapshotClass` CR that consumes the CRDs lives in the `platform-config` pass
|
||||
(`dependsOn: platform`), per the controllers/configs split.
|
||||
- The CDI `StorageProfile`s live one pass further out, in `platform-config-cdi`
|
||||
(`dependsOn: platform-config`) — `storageprofiles.cdi.kubevirt.io` is created by cdi-operator only
|
||||
after it reconciles the `CDI` CR, so they cannot share a pass with it. See
|
||||
`clusters/base/platform-config-cdi.yaml`.
|
||||
|
||||
## Version
|
||||
|
||||
|
||||
@@ -9,8 +9,10 @@
|
||||
# * no VolumeSnapshotClass can be created (item 2), and
|
||||
# * CDI cannot do SNAPSHOT smart-clone (item 3) — it falls back to slow host-assisted `copy`,
|
||||
# which leaves the default virt StorageClass "degraded" (CDIDefaultStorageClassDegraded).
|
||||
# Installing it here (controllers pass) registers the CRDs; the VolumeSnapshotClass + the CDI
|
||||
# StorageProfiles that CONSUME them are applied by platform-config (dependsOn: platform).
|
||||
# Installing it here (controllers pass) registers the CRDs; the VolumeSnapshotClass that CONSUMES them
|
||||
# is applied by platform-config (dependsOn: platform), and the CDI StorageProfiles that set
|
||||
# `cloneStrategy: snapshot` by platform-config-cdi (dependsOn: platform-config — their own CRD is
|
||||
# created by cdi-operator only after the `CDI` CR reconciles; see clusters/base/platform-config-cdi.yaml).
|
||||
#
|
||||
# CONTROLLERS/CONFIGS SPLIT (F-MON1): this is the CONTROLLERS half. The snapshot.storage.k8s.io
|
||||
# CRDs are registered by the `client/config/crd` base below; the snapshot-controller Deployment
|
||||
|
||||
@@ -29,15 +29,37 @@ spec:
|
||||
sources:
|
||||
- gateway-httproute
|
||||
- service
|
||||
# domain-filter MUST be the actual Cloudflare ZONE (keithjackson.dev), not the platform subdomain
|
||||
# (home.keithjackson.dev) — ExternalDNS filters the account's zones by this value, and a subdomain
|
||||
# filter drops the parent zone → "no hosted zone matching record DNS Name" and nothing is created.
|
||||
# Scope stays effectively home.keithjackson.dev because that's the only suffix services request;
|
||||
# policy=sync + registry=txt only ever touch ExternalDNS-OWNED records, never the pre-existing
|
||||
# wildcard or other keithjackson.dev records. (docs/09; resolves the WS-NET TODO below.)
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# CROSS-CLUSTER DNS OWNERSHIP. Read before changing either knob below.
|
||||
#
|
||||
# On 2026-09-04 prod's external-dns DELETED EVERY ONE of dev's public records. Both clusters ran
|
||||
# `policy: sync` against the SAME Cloudflare zone, served the SAME hostnames, and — because
|
||||
# NEITHER set txtOwnerId — both inherited the chart default owner id `default`. Prod therefore
|
||||
# read dev's TXT registry entries as its own, found no local resource claiming those names, and
|
||||
# garbage-collected them. `policy: sync` is correct and stays; what was missing was identity.
|
||||
# Full write-up: infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# TWO INDEPENDENT GUARDS NOW, either of which alone prevents a recurrence:
|
||||
# 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns.
|
||||
# 2. domainFilters — per-environment name scope; prod cannot even see names outside its
|
||||
# own subtree. Defence in depth, added after the incident.
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Which names this instance may manage at all. Cloudflare-provider caveat, empirically hit
|
||||
# 2026-08: this value is used for ZONE selection as well as record matching, and a filter that is
|
||||
# a strict subdomain of the zone dropped the parent zone entirely ("no hosted zone matching record
|
||||
# DNS Name") — so DEV keeps the zone apex `keithjackson.dev`, which is also what dev serves the
|
||||
# apex of. PROD narrows to `prod.home.keithjackson.dev`. If prod's first reconcile logs no
|
||||
# matching zone, that caveat still applies: prod creates NOTHING (fail-closed, never destructive)
|
||||
# and the fallback is to widen prod's EXTERNALDNS_DOMAIN_FILTER back to the zone apex, leaving
|
||||
# txtOwnerId as the sole guard. See clusters/base/cluster-vars.yaml.
|
||||
domainFilters:
|
||||
- "${DNS_ZONE}"
|
||||
policy: sync # create + update + delete records as services come and go
|
||||
- "${EXTERNALDNS_DOMAIN_FILTER}"
|
||||
# The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence
|
||||
# caused the incident. Dev renders `default` (its live records are already registered under that
|
||||
# owner; changing it would orphan them and duplicate every record). Prod renders `prod`.
|
||||
txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}"
|
||||
policy: sync # create + update + delete records as services come and go — but ONLY
|
||||
# records this owner id owns AND this domainFilter admits (see above).
|
||||
# Cloudflare API token: SOPS-encrypted secret, referenced BY NAME only (OQ-4, ADR-0007).
|
||||
env:
|
||||
- name: CF_API_TOKEN
|
||||
|
||||
@@ -13,7 +13,7 @@ spec:
|
||||
- name: https
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: "*.home.keithjackson.dev"
|
||||
hostname: "*.${BASE_DOMAIN}"
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
- name: http-redirect
|
||||
protocol: HTTP
|
||||
port: 80
|
||||
hostname: "*.home.keithjackson.dev"
|
||||
hostname: "*.${BASE_DOMAIN}"
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Selector
|
||||
@@ -46,7 +46,7 @@ spec:
|
||||
- name: https-apex
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: "home.keithjackson.dev"
|
||||
hostname: "${BASE_DOMAIN}"
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
- name: http-apex-redirect
|
||||
protocol: HTTP
|
||||
port: 80
|
||||
hostname: "home.keithjackson.dev"
|
||||
hostname: "${BASE_DOMAIN}"
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Selector
|
||||
|
||||
@@ -10,8 +10,8 @@ metadata:
|
||||
spec:
|
||||
secretName: wildcard-home-tls # consumed by the Gateway HTTPS listener below
|
||||
dnsNames:
|
||||
- "*.home.keithjackson.dev"
|
||||
- "home.keithjackson.dev" # apex for the C2 UI (docs/09)
|
||||
- "*.${BASE_DOMAIN}"
|
||||
- "${BASE_DOMAIN}" # apex for the C2 UI (docs/09)
|
||||
issuerRef:
|
||||
kind: ClusterIssuer
|
||||
# ${ACME_CLUSTER_ISSUER}: dev -> letsencrypt-staging, prod -> letsencrypt-prod (cluster-vars).
|
||||
|
||||
@@ -24,7 +24,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- headlamp.home.keithjackson.dev
|
||||
- headlamp.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -36,63 +36,63 @@ data:
|
||||
---
|
||||
- Platform:
|
||||
- Headlamp:
|
||||
href: https://headlamp.home.keithjackson.dev
|
||||
href: https://headlamp.${BASE_DOMAIN}
|
||||
description: Kubernetes dashboard (per-user OIDC RBAC)
|
||||
icon: mdi-kubernetes
|
||||
- Grafana:
|
||||
href: https://grafana.home.keithjackson.dev
|
||||
href: https://grafana.${BASE_DOMAIN}
|
||||
description: Metrics, logs & dashboards (fronts Prometheus/Loki/Alertmanager)
|
||||
icon: mdi-chart-areaspline
|
||||
- Semaphore:
|
||||
href: https://semaphore.home.keithjackson.dev
|
||||
href: https://semaphore.${BASE_DOMAIN}
|
||||
description: Ansible automation & CI
|
||||
icon: mdi-cog-play
|
||||
- Gitea:
|
||||
href: https://gitea.home.keithjackson.dev
|
||||
href: https://gitea.${BASE_DOMAIN}
|
||||
description: Git hosting & Actions (GitOps source of truth)
|
||||
icon: mdi-git
|
||||
- Keycloak:
|
||||
href: https://keycloak.home.keithjackson.dev
|
||||
href: https://keycloak.${BASE_DOMAIN}
|
||||
description: Identity provider (SSO / OIDC)
|
||||
icon: mdi-account-key
|
||||
|
||||
- Media:
|
||||
- Plex:
|
||||
href: https://plex.home.keithjackson.dev
|
||||
href: https://plex.${BASE_DOMAIN}
|
||||
description: Media streaming
|
||||
icon: mdi-plex
|
||||
- Seerr:
|
||||
href: https://seerr.home.keithjackson.dev
|
||||
href: https://seerr.${BASE_DOMAIN}
|
||||
description: Media requests
|
||||
icon: mdi-play-box-multiple
|
||||
- Sonarr:
|
||||
href: https://sonarr.home.keithjackson.dev
|
||||
href: https://sonarr.${BASE_DOMAIN}
|
||||
description: TV library manager
|
||||
icon: mdi-television-classic
|
||||
- Radarr:
|
||||
href: https://radarr.home.keithjackson.dev
|
||||
href: https://radarr.${BASE_DOMAIN}
|
||||
description: Movie library manager
|
||||
icon: mdi-movie-open
|
||||
- Jackett:
|
||||
href: https://jackett.home.keithjackson.dev
|
||||
href: https://jackett.${BASE_DOMAIN}
|
||||
description: Indexer proxy
|
||||
icon: mdi-magnify
|
||||
- qBittorrent:
|
||||
href: https://qbittorrent.home.keithjackson.dev
|
||||
href: https://qbittorrent.${BASE_DOMAIN}
|
||||
description: Download client (VPN egress)
|
||||
icon: mdi-download-network
|
||||
|
||||
- Network & Infra:
|
||||
- Pi-hole:
|
||||
href: https://pihole.home.keithjackson.dev
|
||||
href: https://pihole.${BASE_DOMAIN}
|
||||
description: Network-wide DNS & ad-blocking
|
||||
icon: mdi-shield-check
|
||||
- UniFi:
|
||||
href: https://unifi.home.keithjackson.dev:11443
|
||||
href: https://unifi.${BASE_DOMAIN}:11443
|
||||
description: Network controller (UOS)
|
||||
icon: mdi-router-network
|
||||
- Nextcloud:
|
||||
href: https://nextcloud.home.keithjackson.dev
|
||||
href: https://nextcloud.${BASE_DOMAIN}
|
||||
description: Files, calendar & contacts
|
||||
icon: mdi-cloud
|
||||
|
||||
@@ -104,7 +104,7 @@ data:
|
||||
href: https://gitea.keithjackson.dev
|
||||
- Auth portal:
|
||||
- abbr: SSO
|
||||
href: https://auth.home.keithjackson.dev
|
||||
href: https://auth.${BASE_DOMAIN}
|
||||
- Reference:
|
||||
- Homepage docs:
|
||||
- abbr: HP
|
||||
|
||||
@@ -54,7 +54,7 @@ spec:
|
||||
# REQUIRED by Homepage >= 1.0 — requests whose Host header is not in this list get 403.
|
||||
# The public apex host (served via the Gateway) + the pod IP (for kubelet probes).
|
||||
- name: HOMEPAGE_ALLOWED_HOSTS
|
||||
value: "home.keithjackson.dev,$(MY_POD_IP):3000"
|
||||
value: "${BASE_DOMAIN},$(MY_POD_IP):3000"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
|
||||
@@ -28,7 +28,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https-apex # the apex listener (wildcard `https` won't match the bare apex)
|
||||
hostnames:
|
||||
- home.keithjackson.dev
|
||||
- ${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- keycloak.home.keithjackson.dev
|
||||
- keycloak.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
poolMaxSize: 20
|
||||
# --- Exposure: TLS at the Gateway; Keycloak serves HTTP and trusts the proxy headers. ---
|
||||
hostname:
|
||||
hostname: https://keycloak.home.keithjackson.dev
|
||||
hostname: https://keycloak.${BASE_DOMAIN}
|
||||
http:
|
||||
httpEnabled: true # TLS is terminated at the Gateway (edge), not in Keycloak
|
||||
proxy:
|
||||
|
||||
@@ -98,9 +98,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:GRAFANA_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://grafana.home.keithjackson.dev/login/generic_oauth
|
||||
- https://grafana.${BASE_DOMAIN}/login/generic_oauth
|
||||
webOrigins:
|
||||
- https://grafana.home.keithjackson.dev
|
||||
- https://grafana.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -125,10 +125,10 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:OAUTH2_PROXY_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://*.home.keithjackson.dev/oauth2/callback
|
||||
- https://auth.home.keithjackson.dev/oauth2/callback
|
||||
- https://*.${BASE_DOMAIN}/oauth2/callback
|
||||
- https://auth.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://*.home.keithjackson.dev
|
||||
- https://*.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -209,9 +209,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:GITEA_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://gitea.home.keithjackson.dev/user/oauth2/keycloak/callback
|
||||
- https://gitea.${BASE_DOMAIN}/user/oauth2/keycloak/callback
|
||||
webOrigins:
|
||||
- https://gitea.home.keithjackson.dev
|
||||
- https://gitea.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -240,9 +240,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:NEXTCLOUD_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://nextcloud.home.keithjackson.dev/apps/user_oidc/code
|
||||
- https://nextcloud.${BASE_DOMAIN}/apps/user_oidc/code
|
||||
webOrigins:
|
||||
- https://nextcloud.home.keithjackson.dev
|
||||
- https://nextcloud.${BASE_DOMAIN}
|
||||
# Post-logout redirect allow-list (#logout "redirect failed"). On sign-out user_oidc redirects the
|
||||
# browser to Keycloak's end_session_endpoint with post_logout_redirect_uri=<Nextcloud base '/'> and
|
||||
# client_id=nextcloud but NO id_token_hint (provider sendIdTokenHint=0), so Keycloak validates the
|
||||
@@ -250,7 +250,7 @@ spec:
|
||||
# "redirect failed". The wildcard covers the base URL and any logout landing path. Keycloak stores
|
||||
# this as the client attribute `post.logout.redirect.uris` (## separates multiple values).
|
||||
attributes:
|
||||
post.logout.redirect.uris: https://nextcloud.home.keithjackson.dev/*
|
||||
post.logout.redirect.uris: https://nextcloud.${BASE_DOMAIN}/*
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -279,9 +279,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:SEMAPHORE_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://semaphore.home.keithjackson.dev/api/auth/oidc/keycloak/redirect
|
||||
- https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect
|
||||
webOrigins:
|
||||
- https://semaphore.home.keithjackson.dev
|
||||
- https://semaphore.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -318,9 +318,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:HEADLAMP_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://headlamp.home.keithjackson.dev/oidc-callback
|
||||
- https://headlamp.${BASE_DOMAIN}/oidc-callback
|
||||
webOrigins:
|
||||
- https://headlamp.home.keithjackson.dev
|
||||
- https://headlamp.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -363,9 +363,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:CRAFTY_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://crafty.home.keithjackson.dev/oauth2/callback
|
||||
- https://crafty.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://crafty.home.keithjackson.dev
|
||||
- https://crafty.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -398,9 +398,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:PIHOLE_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://pihole.home.keithjackson.dev/oauth2/callback
|
||||
- https://pihole.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://pihole.home.keithjackson.dev
|
||||
- https://pihole.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -433,9 +433,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:JACKETT_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://jackett.home.keithjackson.dev/oauth2/callback
|
||||
- https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://jackett.home.keithjackson.dev
|
||||
- https://jackett.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -459,9 +459,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:QBITTORRENT_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://qbittorrent.home.keithjackson.dev/oauth2/callback
|
||||
- https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://qbittorrent.home.keithjackson.dev
|
||||
- https://qbittorrent.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -485,9 +485,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:RADARR_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://radarr.home.keithjackson.dev/oauth2/callback
|
||||
- https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://radarr.home.keithjackson.dev
|
||||
- https://radarr.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -511,9 +511,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:SONARR_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://sonarr.home.keithjackson.dev
|
||||
- https://sonarr.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt VM(s) failing to launch in {{ $labels.namespace }}"
|
||||
description: "One or more VirtualMachineInstances are in the Failed phase in namespace {{ $labels.namespace }} for 5m — a failed VM launch (image import, scheduling, or qemu error). Check `kubectl get vmi -n {{ $labels.namespace }}` and the virt-launcher pod events."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-launch-failed"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-launch-failed"
|
||||
# Crash-loop: the VM's virt-launcher pod is restarting repeatedly (guest kernel panic, OOM, or a
|
||||
# crashing qemu). >3 restarts in 15m is a crash-loop, not a one-off reboot.
|
||||
- alert: KubeVirtVMCrashLooping
|
||||
@@ -42,7 +42,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt VM crash-looping: {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "virt-launcher pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in 15m — the guest VM is crash-looping (panic / OOM / qemu crash)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-crashloop"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-crashloop"
|
||||
# Control plane down: no virt-controller or virt-api replica reports ready, so no VM lifecycle
|
||||
# operations can proceed. The metrics are `kubevirt_virt_{controller,api}_ready_status` (per-pod
|
||||
# 1=ready); sum()==0 means every replica is unready. The absent() clause also fires if the whole
|
||||
@@ -59,7 +59,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt control plane degraded"
|
||||
description: "The KubeVirt control plane (virt-controller/virt-api) is not ready or its metrics are absent for 10m — VM lifecycle operations (create/start/stop) will fail (docs/06)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-control-plane"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-control-plane"
|
||||
# CDI import stuck: a DataVolume import/clone still in flight after 30m blocks the VM it backs.
|
||||
# kubevirt_cdi_import_pods_high_restart counts importer pods stuck restarting; a positive value
|
||||
# sustained means an import is not progressing (metric name confirmed against CDI at bring-up).
|
||||
@@ -71,4 +71,4 @@ spec:
|
||||
annotations:
|
||||
summary: "CDI DataVolume import stuck: {{ $labels.namespace }}/{{ $labels.name }}"
|
||||
description: "A CDI DataVolume import/clone in {{ $labels.namespace }} has not completed after 30m — the VM disk it backs cannot start. Check the importer pod (often an egress/default-deny block on the image source, or a slow mirror)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cdi-import-stuck"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cdi-import-stuck"
|
||||
@@ -1,9 +1,19 @@
|
||||
---
|
||||
# CDI StorageProfiles — the storage "recipes" CDI uses when it provisions/clones VM disk PVCs
|
||||
# (ADR-0005). CONSUME the cdi.kubevirt.io CRD registered by the CDI operator (platform pass), so —
|
||||
# like cdi-cr.yaml — these CRs are applied by the `platform-config` Flux Kustomization
|
||||
# (dependsOn: platform). They live here beside the KubeVirt/CDI component and are referenced in
|
||||
# place from platform-config/kustomization.yaml.
|
||||
# (ADR-0005).
|
||||
#
|
||||
# APPLIED BY `platform-config-cdi`, NOT `platform-config` (moved 2026-09-05). These are SECOND-ORDER
|
||||
# CRs: `storageprofiles.cdi.kubevirt.io` is NOT registered by the CDI operator install (cdi-operator.yaml
|
||||
# ships only `cdis.cdi.kubevirt.io`) — cdi-operator creates it only AFTER it has reconciled the `CDI` CR
|
||||
# that platform-config applies. Flux server-side dry-runs every object in a pass before applying any of
|
||||
# them, so co-locating these with that CR was a TRUE DEADLOCK: unknown kind -> pass aborts -> CR never
|
||||
# applied -> CRD never created -> platform-config never converges -> every tier below it blocked. It was
|
||||
# invisible on dev (CDI CRDs long present) and cost 12h on prod's cold start, 2026-09-05.
|
||||
# They live here beside the KubeVirt/CDI component and are referenced in place from
|
||||
# platform-config-cdi/kustomization.yaml (dependsOn: platform-config, revision-gated).
|
||||
#
|
||||
# `apps-vms` dependsOn platform-config-cdi for these: a DataVolume created before the longhorn-vm
|
||||
# profile below exists is stamped RWO/Filesystem and is permanently not live-migratable.
|
||||
#
|
||||
# CDI AUTO-CREATES a StorageProfile per StorageClass and fills its STATUS (inferred capabilities).
|
||||
# The operator owns `.status`; the USER owns `.spec`. Setting `.spec` here does NOT fight CDI — it
|
||||
|
||||
@@ -8,6 +8,16 @@
|
||||
# (dependsOn: platform, wait) so the CRDs exist BEFORE the CRs apply. Applying both in one atomic pass
|
||||
# deadlocks ("no matches for kind KubeVirt"), exactly the failure F-MON1 documents.
|
||||
#
|
||||
# THERE IS A THIRD PASS FOR CDI, and it is not optional (2026-09-05). The two operator manifests below
|
||||
# register exactly ONE CDI CRD, `cdis.cdi.kubevirt.io`. Every other CDI CRD — `storageprofiles`,
|
||||
# `datavolumes`, `datasources`, `dataimportcrons` — is created by cdi-operator only AFTER it reconciles
|
||||
# the `CDI` CR. So `cdi-storageprofiles.yaml` is a SECOND-ORDER CR and CANNOT share a pass with
|
||||
# `cdi-cr.yaml`: the dry-run of the unknown `StorageProfile` kind aborts the pass, the CR is never
|
||||
# applied, the CRD is never created, and platform-config deadlocks (cost 12h on prod's cold start).
|
||||
# It is therefore applied by a third Kustomization, `platform-config-cdi` (dependsOn platform-config).
|
||||
# Before adding any new CDI CR here, ask which pass owns it: CRD in git -> platform-config;
|
||||
# CRD created by cdi-operator at runtime -> platform-config-cdi.
|
||||
#
|
||||
# Namespaces: the operator manifests each create their own namespace (`kubevirt`, `cdi`) with the
|
||||
# pod-security profile they need (kubevirt is privileged) — so we do NOT redeclare them here (that
|
||||
# would be a duplicate-resource kustomize error). These are PLATFORM namespaces: they intentionally
|
||||
|
||||
@@ -22,14 +22,19 @@ platform/kyverno/
|
||||
│ ├── disallow-plaintext-secrets.yaml # R-SEC-01 (CI-only: sops block required)
|
||||
│ ├── restrict-host-namespaces.yaml # R-SEC-02 (+R-NET-01 bypass guard)
|
||||
│ ├── require-egress-class-label.yaml # R-NET-05
|
||||
│ └── require-auth-mode-label.yaml # R-IDP-02
|
||||
│ ├── require-auth-mode-label.yaml # R-IDP-02
|
||||
│ └── label-cdi-worker-pods.yaml # R-NET-05+R-IDP-02 (mutate: CDI's own worker pods)
|
||||
├── servicedefinition/ # ServiceDefinition shape validation (kyverno-json)
|
||||
│ ├── validate-servicedefinition.yaml # R-GITOPS-02
|
||||
│ ├── payloads/ (valid-* / invalid-*) # pass & fail fixtures
|
||||
│ └── README.md
|
||||
├── tests/ # `kyverno test` suite — runnable L0 tests NOW, no cluster
|
||||
│ ├── kyverno-test.yaml # policies × fixtures × expected results
|
||||
│ └── resources/ (good-* / bad-* fixtures + generated golden)
|
||||
│ ├── resources/ (good-* / bad-* fixtures + generated golden)
|
||||
│ └── controller-worker-pods/ # the CDI-worker mutation, proven BOTH ways
|
||||
│ ├── kyverno-test.yaml # 2 Tests: cdi-sa requester vs ordinary requester
|
||||
│ ├── userinfo-cdi-sa.yaml / userinfo-ordinary.yaml
|
||||
│ └── resources/ + patched/ # real + FORGED importer pods; expected patch
|
||||
├── admission/ # in-cluster enforcing bundle (scaffolding, T-2.7)
|
||||
│ ├── namespace.yaml · helmrelease.yaml · kustomization.yaml
|
||||
│ ├── secret-provenance.yaml # R-SEC-01 admission variant (provenance)
|
||||
@@ -47,6 +52,7 @@ platform/kyverno/
|
||||
| `restrict-host-namespaces` | `disallow-host-namespaces`, `disallow-privileged-containers` | **R-SEC-02** (+R-NET-01) | docs/07 least reach | App pods can't use hostNetwork/PID/IPC or privileged (would bypass default-deny) |
|
||||
| `require-egress-class-label` | `require-egress-class` | **R-NET-05** | docs/03, docs/13 | App pods must declare `egress-class` ∈ {internet,vpn} |
|
||||
| `require-auth-mode-label` | `require-auth-mode` | **R-IDP-02** | docs/13, docs/15 | App pods must declare `auth-mode` ∈ {oidc,forward-auth,native,none} |
|
||||
| `label-cdi-worker-pods` (mutate) | `add-worker-pod-labels` | **R-NET-05**, **R-IDP-02** | docs/03, docs/13 | CDI's **controller-generated** worker pods (`importer-*`/`uploader-*`/`cloner-*`, created by SA `cdi/cdi-sa`) get `egress-class: internet` + `auth-mode: none` **added**, so they satisfy the two policies above instead of being exempted from them. Matched on the **generating ServiceAccount**, never a pod-name prefix |
|
||||
| `validate-servicedefinition` (kyverno-json) | shape / dependencies / storage | **R-GITOPS-02** | docs/13 (+03,15) | ServiceDefinition is schema/invariant valid before render; public⇒not `none` auth |
|
||||
| `admission/secret-provenance` | `secrets-must-be-flux-managed` | **R-SEC-01** | docs/07 | **Admission-only** variant: app-ns Secrets must come from Flux/SOPS (see below) |
|
||||
|
||||
@@ -90,6 +96,19 @@ self-contained — no cluster, no external state (reproducibility contract, R-TE
|
||||
|
||||
- **Label keys are a proposed contract** (`labels.md`) shared with WS-NET / WS-SEC /
|
||||
WS-GITOPS. Values are spec-fixed; keys need owner sign-off (coordination-protocol §2).
|
||||
- **Controller-generated pods are a cold-start hazard.** A policy that matches *any* Pod in an
|
||||
app namespace also matches pods that no manifest in git authors. Prod hit this on 2026-09-06:
|
||||
`require-auth-mode-label` + `require-egress-class-label` denied CDI's own `importer-prime-*`
|
||||
pod, so `DataVolume unifi/unifi-rootdisk` stuck in `ImportScheduled` and the UniFi VM had no
|
||||
disk. Fix: `policies/label-cdi-worker-pods.yaml` **mutates** those pods to carry truthful
|
||||
labels (matched on the generating ServiceAccount `cdi/cdi-sa`), rather than excluding them.
|
||||
`virt-launcher-*` pods need nothing — they inherit the VM pod-template labels. KubeVirt
|
||||
hotplug (`hp-volume-*`) and export (`virt-export-*`) pods do **not** inherit them and are
|
||||
unused here; extend the same policy if either is adopted.
|
||||
- **`PolicyException` is NOT usable in this install**: the Kyverno admission/background/reports
|
||||
controllers all run `--enablePolicyException=false` (chart default, verified on prod
|
||||
2026-09-06). The CRD exists (from the pre-applied CRD bundle) so a `PolicyException` would
|
||||
APPLY cleanly and be silently ignored — a policy hole that looks like a fix and isn't.
|
||||
- **Secrets CI vs admission divergence** is deliberate and documented in
|
||||
`admission/README.md` and `policies/disallow-plaintext-secrets.yaml`.
|
||||
- The ServiceDefinition JSON Schema (WS-GITOPS T-0.3) is the complementary structural
|
||||
|
||||
@@ -13,6 +13,10 @@ resources:
|
||||
- ../policies/restrict-host-namespaces.yaml
|
||||
- ../policies/require-egress-class-label.yaml
|
||||
- ../policies/require-auth-mode-label.yaml
|
||||
# mutate: gives CDI's OWN generated worker pods (importer-*/uploader-*/cloner-*) the two
|
||||
# labels the policies above require -- they are controller-generated, so nothing in git can
|
||||
# label them. Ships with them, never separately.
|
||||
- ../policies/label-cdi-worker-pods.yaml
|
||||
# --- secrets: the admission variant REPLACES the CI-only sops-block policy ---
|
||||
# (../policies/disallow-plaintext-secrets.yaml is CI-only; see secret-provenance.yaml)
|
||||
- secret-provenance.yaml
|
||||
@@ -15,6 +15,17 @@ Shared contract with WS-NET (egress rendering), WS-SEC (auth), and WS-GITOPS (re
|
||||
| `homelab.keithjackson.dev/baseline` | NetworkPolicy | `default-deny` | (convention) identifies the baseline NP | `require-default-deny-baseline` (generate), `validate-default-deny-shape` |
|
||||
| `homelab.keithjackson.dev/secret-managed-by` | Secret | `controller` | (convention) exempts controller-generated Secrets | `disallow-plaintext-secrets` (exclude), admission `secret-provenance` |
|
||||
|
||||
## Controller-generated pods (who sets the labels when git cannot)
|
||||
|
||||
`egress-class` / `auth-mode` are normally projected onto pod templates by the renderer (below).
|
||||
Some pods are generated by a **controller at runtime** and therefore have no manifest in git to
|
||||
label — CDI's `importer-*` / `uploader-*` / `cloner-*` worker pods are the live example: both
|
||||
label policies denied CDI's importer on prod (2026-09-06) and the UniFi VM got no disk. For those,
|
||||
`policies/label-cdi-worker-pods.yaml` **adds** the labels at admission (`+(…)` add-if-absent),
|
||||
matched on the **generating ServiceAccount** (`cdi/cdi-sa`) — never on a pod-name prefix or on
|
||||
CDI's own labels, both of which any pod author in the namespace can copy. `virt-launcher-*` needs
|
||||
nothing: KubeVirt propagates the VM pod-template labels onto it (verified on dev).
|
||||
|
||||
## Rendering expectation (for WS-GITOPS)
|
||||
|
||||
The ServiceDefinition (`docs/13`) declares intent — `egress:` and `auth.mode:` — at the
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Serves: R-NET-05 (egress is a declared property) + R-IDP-02 (auth mode is declared) —
|
||||
# this policy does NOT weaken either requirement. It makes the one class of pod that
|
||||
# CANNOT carry the labels from git SATISFY them, instead of being exempted from them.
|
||||
#
|
||||
# THE DEFECT IT FIXES (observed on LIVE prod, 2026-09-05/06):
|
||||
# `DataVolume unifi/unifi-rootdisk` sat in ImportScheduled for hours holding a detached
|
||||
# 40Gi `prime-…` scratch PVC. `kubectl -n cdi logs deploy/cdi-deployment` showed:
|
||||
# admission webhook "validate.kyverno.svc-fail" denied the request: resource
|
||||
# Pod/unifi/importer-prime-<uid> was blocked due to the following policies
|
||||
# require-auth-mode-label / require-egress-class-label
|
||||
# CDI generates its own short-lived worker pods (importer-*, uploader-*, cloner-*/
|
||||
# clone-source-*) from inside the cdi-deployment controller. Those pods are not in git,
|
||||
# so nothing can put the platform labels on them, and both label policies match ANY Pod
|
||||
# in a namespace labelled homelab.keithjackson.dev/app-namespace=true. Result: the VM
|
||||
# never gets a disk. This is a COLD-START defect, not a prod quirk — dev's identical
|
||||
# import Succeeded before the policies were enforcing, so dev has never exercised it and
|
||||
# would hit it on any rebuild.
|
||||
#
|
||||
# WHY MUTATE (and not an exception/exclusion):
|
||||
# The honest reading of R-NET-05/R-IDP-02 is "every pod's egress path and auth posture is
|
||||
# DECLARED". A CDI importer pod has truthful values for both:
|
||||
# egress-class: internet — it pulls the image from the public internet over https
|
||||
# (cloud-images.ubuntu.com / quay.io) and must take the
|
||||
# default WAN path, which is exactly what Cilium's egress
|
||||
# gateway does for `internet` (ADR-0013/ADR-0015). It is NOT
|
||||
# a `vpn` workload.
|
||||
# auth-mode: none — the pod serves no UI and terminates when the import ends;
|
||||
# there is nothing to put an IdP in front of. docs/13 permits
|
||||
# `none` for internal, non-public workloads (the
|
||||
# ServiceDefinition policy forbids only public + none).
|
||||
# So the requirement is SATISFIED rather than bypassed: the invariant "every pod in an app
|
||||
# namespace carries both labels" stays literally true cluster-wide, both label policies
|
||||
# stay untouched (no `exclude`, no PolicyException, background scanning still on), and the
|
||||
# labels are functional — the pod is now selectable by the Cilium egress fabric instead of
|
||||
# being an unlabelled pod that silently falls through to the default path.
|
||||
#
|
||||
# THE MATCH IS THE GENERATING ServiceAccount, NOT A NAME PREFIX:
|
||||
# `subjects: ServiceAccount cdi/cdi-sa` is the identity in the AdmissionReview's userInfo —
|
||||
# the controller that issued the CREATE. Verified on prod from the Kyverno admission log of
|
||||
# the blocked request:
|
||||
# user={"username":"system:serviceaccount:cdi:cdi-sa", ...} roles=["cdi:cdi-deployment"]
|
||||
# name=importer-prime-be3f9d1c-… namespace=unifi operation=CREATE
|
||||
# A pod NAME (`importer-*`) or CDI's own labels (`cdi.kubevirt.io`) are attacker-CHOOSABLE:
|
||||
# anyone who can create a pod in the namespace can name it `importer-prime-x` and copy the
|
||||
# labels, and would then inherit the exemption. The requesting ServiceAccount cannot be
|
||||
# chosen by the pod author — only by someone who already holds cdi-sa's token, which already
|
||||
# confers "create any pod in any namespace" cluster-wide. So this adds no reachable
|
||||
# privilege. It is also VERSION-ROBUST: it covers every CDI worker kind (importer, uploader,
|
||||
# cloner/clone-source) without enumerating names that change between CDI releases —
|
||||
# confirmed on prod that cdi-sa is the ONLY cdi-namespace ServiceAccount that can create
|
||||
# pods (`kubectl auth can-i create pods -n unifi --as …`: cdi-sa yes; cdi-apiserver no;
|
||||
# cdi-uploadproxy no).
|
||||
#
|
||||
# DELIBERATELY NOT COVERED — KubeVirt's own generated pods:
|
||||
# virt-launcher-* pods INHERIT the VM's pod-template labels, so a VM authored in this repo
|
||||
# already passes (verified on dev: virt-launcher-unifi-22dg7 carries egress-class=internet,
|
||||
# auth-mode=native from apps/vms/unifi/30-virtualmachine.yaml). Exempting
|
||||
# kubevirt-controller would REMOVE that useful enforcement on VM authors, so it is not done.
|
||||
# KubeVirt's hotplug (`hp-volume-*`) and export (`virt-export-*`) pods do NOT inherit those
|
||||
# labels, but nothing in this repo uses volume hotplug or VirtualMachineExport; if either is
|
||||
# adopted, extend this policy then — with the same ServiceAccount-scoped shape.
|
||||
#
|
||||
# OPERATOR NOTE — this is the cluster's FIRST resource-MUTATING policy. Verified on prod
|
||||
# 2026-09-06 that `kyverno-resource-mutating-webhook-cfg` currently holds 0 webhooks; with the
|
||||
# chart's `autoUpdateWebhooks=true` Kyverno registers `mutate.kyverno.svc-fail` for Pods within
|
||||
# seconds of this policy being applied. Prod runs `forceFailurePolicyIgnore=false`, so that
|
||||
# webhook is failurePolicy: Fail — the same posture as the validating webhook that is ALREADY in
|
||||
# the path for these pods, so no new fail-open surface and no new blocking surface. CDI retries
|
||||
# the importer pod on a loop, so the stuck import proceeds once the policy lands (the detached
|
||||
# `prime-…` scratch PVC left behind by the blocked attempts is a separate operator cleanup).
|
||||
#
|
||||
# RESIDUAL TRADE-OFF (state it plainly, it is reversible by deleting this file):
|
||||
# Anything created by `cdi:cdi-sa` in an app namespace now gets `egress-class: internet` +
|
||||
# `auth-mode: none` supplied for it instead of being rejected for not declaring them. Whoever
|
||||
# holds cdi-sa's token can therefore run a pod in an app namespace without declaring its egress
|
||||
# class. That identity can already create arbitrary pods in every namespace (ClusterRole `cdi`:
|
||||
# pods create/delete cluster-wide), so it is not a new capability — but the declaration is now
|
||||
# assumed rather than demanded for that one identity. Nothing else changes: the pod is still
|
||||
# subject to restrict-host-namespaces, the namespace default-deny baseline, and every other
|
||||
# policy, and `internet` is the LEAST-reach of the two egress classes (it does not grant the VPN
|
||||
# path).
|
||||
#
|
||||
# PROOF (offline, no cluster — platform/kyverno/tests/controller-worker-pods/):
|
||||
# (a) a CDI importer pod created by cdi:cdi-sa is mutated and then PASSES both policies;
|
||||
# (b) the SAME namespace's ordinary pod — including one that FORGES the importer name and
|
||||
# CDI's labels — is NOT mutated and is still DENIED by both policies.
|
||||
apiVersion: kyverno.io/v1
|
||||
kind: ClusterPolicy
|
||||
metadata:
|
||||
name: label-cdi-worker-pods
|
||||
annotations:
|
||||
policies.kyverno.io/title: Label CDI-generated worker pods
|
||||
policies.kyverno.io/category: Network Isolation
|
||||
policies.kyverno.io/severity: medium
|
||||
homelab.keithjackson.dev/requirements: R-NET-05,R-IDP-02
|
||||
policies.kyverno.io/description: >-
|
||||
Adds homelab.keithjackson.dev/egress-class=internet and auth-mode=none to the
|
||||
short-lived worker Pods that the CDI controller (ServiceAccount cdi/cdi-sa)
|
||||
generates in application namespaces, so they satisfy require-egress-class-label
|
||||
and require-auth-mode-label instead of being exempted from them.
|
||||
spec:
|
||||
# No validationFailureAction: this is a MUTATE policy, there is nothing to fail.
|
||||
# background MUST be false: `match.subjects` is userInfo, which exists only on an
|
||||
# AdmissionReview. Kyverno rejects a userInfo match with background scanning enabled
|
||||
# ("invalid variable used at path ... policy contains 'userInfo'"), and a background scan
|
||||
# of an already-created pod has no requester to match anyway.
|
||||
background: false
|
||||
rules:
|
||||
- name: add-worker-pod-labels
|
||||
match:
|
||||
# One filter entry: the resource conditions AND the subject must both match.
|
||||
all:
|
||||
- resources:
|
||||
kinds:
|
||||
- Pod
|
||||
operations:
|
||||
- CREATE
|
||||
namespaceSelector:
|
||||
matchLabels:
|
||||
homelab.keithjackson.dev/app-namespace: "true"
|
||||
subjects:
|
||||
# The CDI controller (cdi-deployment) ServiceAccount — the identity that
|
||||
# CREATEs importer-*/uploader-*/cloner-* pods. Not the pods' own SA.
|
||||
- kind: ServiceAccount
|
||||
namespace: cdi
|
||||
name: cdi-sa
|
||||
mutate:
|
||||
patchStrategicMerge:
|
||||
metadata:
|
||||
labels:
|
||||
# `+(key)` is Kyverno's ADD-IF-ABSENT anchor: an explicit value already on the
|
||||
# pod is never overwritten (so a future CDI worker that legitimately needs
|
||||
# egress-class: vpn can be given it, and virt-launcher-style inherited labels
|
||||
# would survive).
|
||||
+(homelab.keithjackson.dev/egress-class): internet
|
||||
+(homelab.keithjackson.dev/auth-mode): none
|
||||
@@ -0,0 +1,91 @@
|
||||
# L0 policy-as-code suite for the CDI worker-pod label mutation — runs in CI WITHOUT a cluster
|
||||
# (discovered recursively by the gate's `kyverno test platform/kyverno`; ci/l0.sh).
|
||||
#
|
||||
# WHY A SEPARATE SUITE from ../kyverno-test.yaml: a `kyverno test` manifest carries ONE
|
||||
# `userinfo`, and the whole point here is that the SAME pods get different outcomes for
|
||||
# different requesters. Two Test documents, two identities, one file:
|
||||
# 1. cdi-controller-worker-pods-admitted (requester = cdi/cdi-sa) -> mutated, ADMITTED
|
||||
# 2. cdi-worker-exemption-is-narrow (requester = ordinary user) -> not mutated, DENIED
|
||||
# Keeping the mutate policy OUT of ../kyverno-test.yaml is deliberate: that suite declares no
|
||||
# userinfo, and with no userinfo the offline engine does not discriminate on `subjects`, which
|
||||
# would silently turn its negative (result: fail) rows into passes.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cdi-controller-worker-pods-admitted
|
||||
# Namespace labels for offline namespaceSelector resolution (shared with the main suite).
|
||||
variables: ../values.yaml
|
||||
userinfo: userinfo-cdi-sa.yaml
|
||||
policies:
|
||||
- ../../policies/label-cdi-worker-pods.yaml
|
||||
- ../../policies/require-egress-class-label.yaml
|
||||
- ../../policies/require-auth-mode-label.yaml
|
||||
resources:
|
||||
- ../resources/namespaces.yaml
|
||||
- resources/cdi-worker-pods.yaml
|
||||
results:
|
||||
# ---- the mutation fires for the CDI controller and adds exactly the two labels ----
|
||||
- policy: label-cdi-worker-pods
|
||||
rule: add-worker-pod-labels
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
patchedResources: patched/importer-mutated.yaml
|
||||
result: pass
|
||||
# ---- R-NET-05 + R-IDP-02: the mutated CDI worker pod now SATISFIES both policies, so the
|
||||
# import CDI could not start is admitted. (The engine validates the mutated resource.)
|
||||
- policy: require-egress-class-label
|
||||
rule: require-egress-class
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
result: pass
|
||||
- policy: require-auth-mode-label
|
||||
rule: require-auth-mode
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
result: pass
|
||||
---
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cdi-worker-exemption-is-narrow
|
||||
variables: ../values.yaml
|
||||
userinfo: userinfo-ordinary.yaml
|
||||
policies:
|
||||
- ../../policies/label-cdi-worker-pods.yaml
|
||||
- ../../policies/require-egress-class-label.yaml
|
||||
- ../../policies/require-auth-mode-label.yaml
|
||||
resources:
|
||||
- ../resources/namespaces.yaml
|
||||
- resources/cdi-worker-pods.yaml
|
||||
results:
|
||||
# ---- the mutation does NOT fire for an ordinary requester, not even for a pod that forges
|
||||
# the importer's name and CDI's labels ----
|
||||
- policy: label-cdi-worker-pods
|
||||
rule: add-worker-pod-labels
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: skip
|
||||
# ---- so every unlabelled pod in the SAME app namespace is still DENIED (R-NET-05) ----
|
||||
- policy: require-egress-class-label
|
||||
rule: require-egress-class
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: fail
|
||||
# ---- ... and by R-IDP-02 ----
|
||||
- policy: require-auth-mode-label
|
||||
rule: require-auth-mode
|
||||
kind: Pod
|
||||
resources:
|
||||
- importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
- importer-prime-forged-by-an-app
|
||||
- plain-app-pod-no-labels
|
||||
result: fail
|
||||
@@ -0,0 +1,16 @@
|
||||
# Expected result of the mutation: CDI's own labels untouched, the two platform labels ADDED.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: none
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: importer
|
||||
image: quay.io/kubevirt/cdi-importer:v1.62.0
|
||||
@@ -0,0 +1,53 @@
|
||||
# Fixtures for the CDI-worker-pod label mutation (policies/label-cdi-worker-pods.yaml).
|
||||
#
|
||||
# All three pods live in `app-ns` (../resources/namespaces.yaml — carries
|
||||
# homelab.keithjackson.dev/app-namespace: "true"), so all three are IN SCOPE of
|
||||
# require-egress-class-label + require-auth-mode-label. What separates them is WHO creates
|
||||
# them, which is the whole point of the mechanism: the two "forged" pods are byte-for-byte
|
||||
# as CDI-looking as the real one.
|
||||
#
|
||||
# ---- (a) the REAL CDI worker pod: shape copied from the pod CDI actually tried to create on
|
||||
# prod (name importer-prime-<pvc-uid>, CDI's own labels, no platform labels). It is
|
||||
# created by ServiceAccount cdi/cdi-sa — see userinfo-cdi-sa.yaml.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-be3f9d1c-c3ce-455a-a450-3d9e95c7baa2
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
# NOTE: no homelab.keithjackson.dev/* labels — CDI generates this pod itself, nothing in
|
||||
# git can label it. This is exactly why the import was blocked.
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: importer
|
||||
image: quay.io/kubevirt/cdi-importer:v1.62.0
|
||||
---
|
||||
# ---- (b) FORGED: an ordinary pod that copies the importer's NAME PREFIX and CDI's labels.
|
||||
# Proves that a name-prefix or label-based exemption would have been walkable, and
|
||||
# that this ServiceAccount-scoped one is not.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: importer-prime-forged-by-an-app
|
||||
namespace: app-ns
|
||||
labels:
|
||||
app: containerized-data-importer
|
||||
cdi.kubevirt.io: importer
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: registry.example/app:1.0.0
|
||||
---
|
||||
# ---- (c) a plain unlabelled application pod in the SAME namespace. Must still be DENIED.
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: plain-app-pod-no-labels
|
||||
namespace: app-ns
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: registry.example/app:1.0.0
|
||||
@@ -0,0 +1,21 @@
|
||||
# The requester identity Kyverno actually saw on prod when CDI created the importer pod
|
||||
# (from the kyverno-admission-controller log of the blocked request, 2026-09-06):
|
||||
# user={"username":"system:serviceaccount:cdi:cdi-sa","groups":["system:serviceaccounts",
|
||||
# "system:serviceaccounts:cdi","system:authenticated"], ...}
|
||||
# roles=["cdi:cdi-deployment"] clusterroles=["cdi", …]
|
||||
# `kyverno test` feeds this to the engine as the AdmissionReview userInfo so
|
||||
# `match.subjects` can be evaluated offline.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: UserInfo
|
||||
metadata:
|
||||
name: cdi-controller
|
||||
clusterRoles:
|
||||
- cdi
|
||||
roles:
|
||||
- cdi:cdi-deployment
|
||||
userInfo:
|
||||
username: system:serviceaccount:cdi:cdi-sa
|
||||
groups:
|
||||
- system:serviceaccounts
|
||||
- system:serviceaccounts:cdi
|
||||
- system:authenticated
|
||||
@@ -0,0 +1,16 @@
|
||||
# An ORDINARY requester in the application namespace — the attacker/mistake case. Anything
|
||||
# that is not cdi/cdi-sa: an app's own ServiceAccount, a human's kubectl, Flux.
|
||||
# The mutation must NOT fire for this identity, so both label policies must still DENY —
|
||||
# even for a pod that forged the importer's name and CDI's labels.
|
||||
apiVersion: cli.kyverno.io/v1alpha1
|
||||
kind: UserInfo
|
||||
metadata:
|
||||
name: ordinary-namespace-user
|
||||
clusterRoles:
|
||||
- edit
|
||||
userInfo:
|
||||
username: system:serviceaccount:app-ns:default
|
||||
groups:
|
||||
- system:serviceaccounts
|
||||
- system:serviceaccounts:app-ns
|
||||
- system:authenticated
|
||||
@@ -38,7 +38,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Node exporter down on {{ $labels.instance }}"
|
||||
description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-down"
|
||||
- alert: NodeMemoryPressureHigh
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
|
||||
for: 10m
|
||||
@@ -47,7 +47,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Memory pressure high on {{ $labels.instance }}"
|
||||
description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-memory"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-memory"
|
||||
- alert: NodeDiskFillingUp
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}
|
||||
@@ -58,7 +58,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
||||
description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-disk"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-disk"
|
||||
- alert: NodeCPUPressureHigh
|
||||
# PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to
|
||||
# run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU
|
||||
@@ -71,7 +71,7 @@ spec:
|
||||
annotations:
|
||||
summary: "CPU pressure high on {{ $labels.instance }}"
|
||||
description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu"
|
||||
- alert: NodeCPULoadHigh
|
||||
# 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more
|
||||
# runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node.
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Load per core high on {{ $labels.instance }}"
|
||||
description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu"
|
||||
|
||||
# =====================================================================================
|
||||
# Storage — Longhorn
|
||||
@@ -113,7 +113,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume degraded: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-degraded"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-degraded"
|
||||
- alert: LonghornVolumeFaulted
|
||||
expr: longhorn_volume_robustness == 3
|
||||
for: 1m
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume FAULTED: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-faulted"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-faulted"
|
||||
- alert: LonghornVolumeCapacityHigh
|
||||
expr: (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
@@ -131,7 +131,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume near capacity: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is over 85% used for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity"
|
||||
- alert: LonghornNodeStorageHigh
|
||||
expr: (longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
@@ -140,7 +140,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn node storage near capacity: {{ $labels.node }}"
|
||||
description: "Longhorn node {{ $labels.node }} is over 85% of its storage capacity for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity"
|
||||
|
||||
# =====================================================================================
|
||||
# GitOps — Flux
|
||||
@@ -169,7 +169,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-reconcile"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-reconcile"
|
||||
- alert: FluxResourceSuspended
|
||||
expr: |
|
||||
max by (namespace, name, kind) (gotk_suspend_status) == 1
|
||||
@@ -179,7 +179,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-suspended"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-suspended"
|
||||
|
||||
# =====================================================================================
|
||||
# Certificates — cert-manager
|
||||
@@ -205,7 +205,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate expiring soon: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry"
|
||||
- alert: CertificateExpiryImminent
|
||||
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600)
|
||||
for: 1h
|
||||
@@ -214,7 +214,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate expiry imminent: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry"
|
||||
- alert: CertificateNotReady
|
||||
expr: certmanager_certificate_ready_status{condition="False"} == 1
|
||||
for: 1h
|
||||
@@ -223,7 +223,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate not ready / issuance failing: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-issuance"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-issuance"
|
||||
|
||||
# =====================================================================================
|
||||
# Networking — VPN / egress-gateway / denied flows (Hubble)
|
||||
@@ -255,7 +255,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Egress VPN tunnel down"
|
||||
description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-down"
|
||||
# (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun
|
||||
# retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd
|
||||
# service, steering is Cilium eBPF). Tunnel + gateway-node health is covered by `VPNTunnelDown`
|
||||
@@ -270,7 +270,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Spike in policy-denied network flows"
|
||||
description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/denied-flows"
|
||||
|
||||
# =====================================================================================
|
||||
# Backups — Velero
|
||||
@@ -296,7 +296,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Velero backup failed: schedule {{ $labels.schedule }}"
|
||||
description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-failure"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure"
|
||||
- alert: VeleroBackupPartialFailure
|
||||
expr: increase(velero_backup_partial_failure_total[1h]) > 0
|
||||
for: 5m
|
||||
@@ -305,7 +305,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Velero backup partially failed"
|
||||
description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-partial"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial"
|
||||
- alert: VeleroNoRecentBackup
|
||||
expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600)
|
||||
for: 1h
|
||||
@@ -314,7 +314,7 @@ spec:
|
||||
annotations:
|
||||
summary: "No successful Velero backup in 24h: {{ $labels.schedule }}"
|
||||
description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-missed"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed"
|
||||
|
||||
# =====================================================================================
|
||||
# Identity — Keycloak (SSO is a critical dependency — docs/15)
|
||||
@@ -345,7 +345,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Keycloak IdP down"
|
||||
description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-down"
|
||||
- alert: KeycloakDatabaseDown
|
||||
# Keycloak's PostgreSQL runs as a CloudNativePG Cluster `keycloak-db` (pods keycloak-db-N).
|
||||
# DR-critical, Longhorn-backed. (CNPG also exports cnpg_* metrics via its PodMonitor.)
|
||||
@@ -358,7 +358,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Keycloak database unavailable"
|
||||
description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-db"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-db"
|
||||
|
||||
# =====================================================================================
|
||||
# The monitor's own capacity — don't let the monitor blind itself (docs/10)
|
||||
@@ -386,7 +386,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}"
|
||||
description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/monitoring-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/monitoring-capacity"
|
||||
- alert: PrometheusTargetsMissing
|
||||
expr: |
|
||||
(count(up == 0) / count(up)) > 0.10
|
||||
@@ -396,7 +396,7 @@ spec:
|
||||
annotations:
|
||||
summary: "More than 10% of Prometheus targets are down"
|
||||
description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/targets-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/targets-down"
|
||||
|
||||
# =====================================================================================
|
||||
# Automated updates — Flux image-automation + hands-off update health (docs/11, ADR-0009)
|
||||
@@ -430,7 +430,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux image-update-automation failing: {{ $labels.name }}"
|
||||
description: "ImageUpdateAutomation {{ $labels.namespace }}/{{ $labels.name }} cannot reconcile (git write-back) for 30m — stable image advances have stopped (R-UPD-02). Check the git write credential."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/image-automation"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/image-automation"
|
||||
# An ImagePolicy/ImageRepository can't resolve the latest stable tag (registry rate-limit /
|
||||
# auth / bad range) → that service stops receiving updates even though automation is healthy.
|
||||
- alert: ImagePolicyStalled
|
||||
@@ -444,7 +444,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Image scan/policy stalled: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} cannot resolve its stable tag for 1h (registry rate-limit / creds / range) — no longer receiving updates (R-UPD-01)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/image-policy-stalled"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/image-policy-stalled"
|
||||
# A bad update was health-gated and AUTO-ROLLED-BACK (or is stuck mid-remediation). helm-controller
|
||||
# holds Ready=False while a failed upgrade remediates/rolls back to the last-good revision
|
||||
# (HelmRelease.spec.upgrade.remediation + .spec.rollback). This is the R-UPD-03 safety rail firing.
|
||||
@@ -461,4 +461,4 @@ spec:
|
||||
annotations:
|
||||
summary: "Update health-gate failed / rolled back: {{ $labels.name }}"
|
||||
description: "HelmRelease {{ $labels.namespace }}/{{ $labels.name }} failed its health-gated upgrade and is rolling back to the last-good revision (bad-update auto-rollback — R-UPD-03). Consider an `update.pin` escape hatch (docs/11)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/update-rollback"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/update-rollback"
|
||||
@@ -14,7 +14,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- grafana.home.keithjackson.dev
|
||||
- grafana.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -194,15 +194,15 @@ spec:
|
||||
optional: true
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.home.keithjackson.dev
|
||||
root_url: https://grafana.${BASE_DOMAIN}
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Keycloak
|
||||
client_id: grafana
|
||||
scopes: "openid email profile"
|
||||
auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth
|
||||
token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token
|
||||
api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo
|
||||
auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth
|
||||
token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token
|
||||
api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo
|
||||
login_attribute_path: preferred_username
|
||||
email_attribute_path: email
|
||||
# Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
|
||||
|
||||
@@ -32,7 +32,7 @@ spec:
|
||||
existingSecret: oauth2-proxy-credentials
|
||||
extraArgs:
|
||||
provider: oidc
|
||||
oidc-issuer-url: https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
oidc-issuer-url: https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
# Forward-auth (ext-auth) mode: no upstream app; return 202 for an authenticated session so the
|
||||
# Gateway ext-authz can allow/deny. Auth headers are surfaced to the backend.
|
||||
reverse-proxy: "true"
|
||||
@@ -41,9 +41,9 @@ spec:
|
||||
pass-access-token: "true"
|
||||
email-domain: "*"
|
||||
# Single sign-in/callback host for all forward-auth-protected services on the platform zone.
|
||||
redirect-url: https://auth.home.keithjackson.dev/oauth2/callback
|
||||
whitelist-domain: ".home.keithjackson.dev"
|
||||
cookie-domain: ".home.keithjackson.dev"
|
||||
redirect-url: https://auth.${BASE_DOMAIN}/oauth2/callback
|
||||
whitelist-domain: ".${BASE_DOMAIN}"
|
||||
cookie-domain: ".${BASE_DOMAIN}"
|
||||
cookie-secure: "true"
|
||||
# Expose group membership so per-service group authorization (R-IDP-06) can be enforced. The
|
||||
# Keycloak `oauth2-proxy` client emits a `groups` claim (full.path=false => bare names like
|
||||
|
||||
@@ -14,7 +14,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- auth.home.keithjackson.dev
|
||||
- auth.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
---
|
||||
# ============================================================================
|
||||
# Cilium Egress Gateway steering for `egress-class=vpn` pods — ADR-0015.
|
||||
# Serves: R-NET-05, R-NET-06, R-NET-07.
|
||||
# Serves: R-NET-04, R-NET-05, R-NET-06, R-NET-07.
|
||||
# R-NET-04 added 2026-09-06: this file IS the R-NET-04 datapath — the rule that selects between the
|
||||
# two egress paths. Omitting it made `run.sh --status` report R-NET-04 as "TEST-ONLY / impl none"
|
||||
# even though the implementation is right here.
|
||||
#
|
||||
# ##########################################################################
|
||||
# ## LIVE (2026-08-24): wired into kustomization.yaml, reconciled by Flux ##
|
||||
@@ -35,7 +38,7 @@ kind: CiliumEgressGatewayPolicy
|
||||
metadata:
|
||||
name: vpn-egress-steering
|
||||
annotations:
|
||||
homelab.keithjackson.dev/requirements: R-NET-05,R-NET-06,R-NET-07
|
||||
homelab.keithjackson.dev/requirements: R-NET-04,R-NET-05,R-NET-06,R-NET-07
|
||||
homelab.keithjackson.dev/adr: ADR-0015
|
||||
spec:
|
||||
# --- Client selection (INVARIANTS 1 & 2) --------------------------------
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "DNS label -> <name>.home.keithjackson.dev (docs/13, docs/09). Must be a valid RFC 1123 DNS label.",
|
||||
"description": "DNS label -> <name>.${BASE_DOMAIN}, i.e. <name>.home.keithjackson.dev on dev and <name>.prod.home.keithjackson.dev on prod (docs/13, docs/09). Must be a valid RFC 1123 DNS label.",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
|
||||
"minLength": 1,
|
||||
"maxLength": 63
|
||||
@@ -223,8 +223,8 @@
|
||||
"properties": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": "FQDN under the platform zone. Must be a <label>.home.keithjackson.dev name (docs/09).",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.home\\.keithjackson\\.dev$"
|
||||
"description": "Hostname under the platform zone, as a single DNS label plus the ${BASE_DOMAIN} token — e.g. 'gitea.${BASE_DOMAIN}' (docs/09). ${BASE_DOMAIN} is the per-environment hostname suffix from clusters/base/cluster-vars.yaml, resolved by Flux postBuild substitution at reconcile time (dev: home.keithjackson.dev, prod: prod.home.keithjackson.dev). AUTHOR THE TOKEN, NOT A LITERAL: a literal suffix is what let prod be authored to serve dev's public hostnames out of the shared Cloudflare zone, and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10). The literal 'home.keithjackson.dev' form stays accepted so the L1 render fixtures — which are renderer INPUTS, not reconciled manifests — keep validating unchanged.",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.(\\$\\{BASE_DOMAIN\\}|home\\.keithjackson\\.dev)$"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
|
||||
Reference in new issue
Block a user