Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5
|
9137bbd0c3 | ||
|
jakman85andClaude Opus 5
|
c8c1d2935d | ||
|
jakman85andClaude Opus 5
|
9b8979a427 |
No files matched your search
@@ -102,7 +102,7 @@ spec:
|
||||
userData: |-
|
||||
#cloud-config
|
||||
hostname: unifi
|
||||
fqdn: unifi.home.keithjackson.dev
|
||||
fqdn: unifi.${BASE_DOMAIN}
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
packages: [qemu-guest-agent, podman, uidmap, slirp4netns, ca-certificates, curl, jq, cloud-guest-utils]
|
||||
|
||||
@@ -46,7 +46,7 @@ metadata:
|
||||
lbipam.cilium.io/ips: "${UNIFI_L4_VIP}"
|
||||
# external-dns (source: service) publishes A unifi.home.keithjackson.dev -> the pinned VIP.
|
||||
# This is the browser + device target; UI is https://unifi.home.keithjackson.dev:11443.
|
||||
external-dns.alpha.kubernetes.io/hostname: "unifi.home.keithjackson.dev"
|
||||
external-dns.alpha.kubernetes.io/hostname: "unifi.${BASE_DOMAIN}"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -90,9 +90,9 @@ spec:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN
|
||||
value: gitea.home.keithjackson.dev
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.home.keithjackson.dev/
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED
|
||||
value: 'true'
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL
|
||||
@@ -127,7 +127,7 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\
|
||||
\ break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n# A postStart hook MUST exit 0 \u2014 a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff). This hook already ends 0 on both paths, but\n# it is made EXPLICIT so the guarantee does not depend on whatever statement happens to\n# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,\n# the moment its OIDC guard was fixed and the block actually ran for the first time.\nexit 0\n"
|
||||
volumes:
|
||||
- name: data
|
||||
@@ -207,7 +207,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- gitea.home.keithjackson.dev
|
||||
- gitea.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:9117
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://jackett.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -273,7 +273,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- jackett.home.keithjackson.dev
|
||||
- jackett.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:80
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://pihole.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://pihole.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -229,7 +229,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- pihole.home.keithjackson.dev
|
||||
- pihole.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -183,7 +183,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- plex.home.keithjackson.dev
|
||||
- plex.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8080
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://qbittorrent.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -273,7 +273,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- qbittorrent.home.keithjackson.dev
|
||||
- qbittorrent.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -165,7 +165,7 @@ spec:
|
||||
exec /sbin/tini -- /opt/act/run.sh
|
||||
env:
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: https://gitea.home.keithjackson.dev
|
||||
value: https://gitea.${BASE_DOMAIN}
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -104,13 +104,13 @@ spec:
|
||||
name: nextcloud-bootstrap
|
||||
key: admin-password
|
||||
- name: NEXTCLOUD_TRUSTED_DOMAINS
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEHOST
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEPROTOCOL
|
||||
value: https
|
||||
- name: OVERWRITECLIURL
|
||||
value: https://nextcloud.home.keithjackson.dev
|
||||
value: https://nextcloud.${BASE_DOMAIN}
|
||||
- name: NEXTCLOUD_OIDC_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -125,9 +125,9 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"\
|
||||
[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"\
|
||||
$ADMIN_SRC_GROUP\"); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n"
|
||||
- "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[nextcloud-oidc]\
|
||||
\ client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"$ADMIN_SRC_GROUP\"\
|
||||
); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\nif [ \"$p\" -ge 15 ]; then\n nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\nfi\n# A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:\n# the final statement used to be `[ \"$p\" -ge 15 ] && nextcloud_oidc_fail ...`, so on the\n# SUCCESS path the test returned 1 and became the script's exit status. That was latent for\n# as long as the OIDC guard above was dead code and exited 0 first; fixing the guard\n# exposed it immediately. Every failure path here logs loudly and writes a marker file and\n# deliberately does NOT crash the pod \u2014 the break-glass local admin must stay usable.\nexit 0\n"
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
@@ -228,7 +228,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- nextcloud.home.keithjackson.dev
|
||||
- nextcloud.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:7878
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://radarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -295,7 +295,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- radarr.home.keithjackson.dev
|
||||
- radarr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -110,7 +110,7 @@ spec:
|
||||
- name: SEMAPHORE_ADMIN_NAME
|
||||
value: Semaphore Admin
|
||||
- name: SEMAPHORE_ADMIN_EMAIL
|
||||
value: admin@home.keithjackson.dev
|
||||
value: admin@${BASE_DOMAIN}
|
||||
- name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -231,7 +231,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- semaphore.home.keithjackson.dev
|
||||
- semaphore.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -91,16 +91,16 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
- --oidc-groups-claim=groups
|
||||
- --allowed-group=media-users
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
@@ -295,7 +295,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- sonarr.home.keithjackson.dev
|
||||
- sonarr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -223,7 +223,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- seerr.home.keithjackson.dev
|
||||
- seerr.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint for the dev cluster.
|
||||
# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint. ENVIRONMENT-AGNOSTIC: which overlay
|
||||
# it applies is FLUX_ENV_PATH (default clusters/dev). roles/flux_bootstrap exports that from
|
||||
# `flux_bootstrap_env_path`, overridden to clusters/prod in group_vars/k3s_prod/vars.yml — so the
|
||||
# same script bootstraps dev and prod with no fork (R-ENV-02).
|
||||
#
|
||||
# Phase 1, T-1.4 · ADR-0004 (Flux) · ADR-0007 (SOPS+age) · R-GITOPS-01 · R-SEC-01.
|
||||
# =============================================================================
|
||||
@@ -33,7 +36,7 @@
|
||||
# 2. sops-age Secret — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so
|
||||
# decryption works on first pass (../sops/age-key-secret.md). This script
|
||||
# only VERIFIES it exists; it never handles key material.
|
||||
# 3. `kubectl apply -k clusters/dev` — materialises the authored reconciliation graph:
|
||||
# 3. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph:
|
||||
# GitRepository `flux-system` (source) -> Kustomization `platform` ->
|
||||
# `platform-config` -> the app-wave Kustomizations `apps-wave-0/1/2`
|
||||
# (chained dependsOn) + `apps-vms`, with cluster-vars overlay
|
||||
@@ -52,7 +55,10 @@
|
||||
set -euo pipefail
|
||||
|
||||
# --- config (env-overridable) ------------------------------------------------
|
||||
FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}" # dev is the ONLY Phase-1 build target
|
||||
# Which environment overlay to apply. Dev is the DEFAULT (it was the only Phase-1 build target);
|
||||
# prod passes clusters/prod via roles/flux_bootstrap. Changing the default would silently retarget
|
||||
# every existing caller — override it, never edit it.
|
||||
FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}"
|
||||
FLUX_NAMESPACE="${FLUX_NAMESPACE:-flux-system}"
|
||||
SOPS_SECRET_NAME="${SOPS_SECRET_NAME:-sops-age}"
|
||||
# The two Phase-4 image-automation controllers are installed alongside the default set so their CRDs
|
||||
@@ -62,7 +68,7 @@ FLUX_COMPONENTS_EXTRA="${FLUX_COMPONENTS_EXTRA:-image-reflector-controller,image
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/../.." && pwd))"
|
||||
|
||||
echo "==> Flux + SOPS bootstrap (dev) — path=${FLUX_ENV_PATH}"
|
||||
echo "==> Flux + SOPS bootstrap — path=${FLUX_ENV_PATH}"
|
||||
|
||||
# --- 0. tooling present (from infra/devcontainer; versions pinned there) ------
|
||||
for bin in flux kubectl; do
|
||||
@@ -134,7 +140,7 @@ if ! kubectl -n "${FLUX_NAMESPACE}" get secret "${SOPS_SECRET_NAME}" >/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- 3. apply the authored dev entrypoint ------------------------------------
|
||||
# --- 3. apply the authored environment entrypoint ----------------------------
|
||||
echo "==> [3/3] kubectl apply -k ${FLUX_ENV_PATH}"
|
||||
kubectl apply -k "${repo_root}/${FLUX_ENV_PATH}"
|
||||
|
||||
|
||||
@@ -1,11 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
# bootstrap/gitea/bootstrap-gitea.sh — run a SINGLE Gitea container on the HOST (outside the
|
||||
# cluster) as the Phase-1 GitOps bootstrap source. While the cluster is built, Flux (in-cluster)
|
||||
# reconciles this repo from the host Gitea over the k3s-dev NAT gateway. This is the "local
|
||||
# bootstrap source" that migrates to in-cluster Gitea in Phase 6 (docs/05 "Source-of-truth",
|
||||
# CONTRIBUTING.md).
|
||||
# cluster) as the GitOps bootstrap source. While a cluster is built, Flux (in-cluster) reconciles
|
||||
# this repo from the host Gitea. This is the "local bootstrap source" that migrates to in-cluster
|
||||
# Gitea at cutover (docs/05 "Source-of-truth", CONTRIBUTING.md). Dev has already cut over; PROD is
|
||||
# using it now.
|
||||
#
|
||||
# Run by the OPERATOR at bring-up — NOT during author-only, NOT by any agent.
|
||||
# HOW THE CLUSTER REACHES THE HOST — read this before setting HOST_NAT_IP.
|
||||
# It is an ADDRESS OF THIS HOST as seen from the cluster nodes, and it differs per environment:
|
||||
#
|
||||
# PROD (bare metal, current use): HOST_NAT_IP=192.168.2.132
|
||||
# The prod nodes (192.168.2.10/.11/.12) and this workstation are on the SAME LAN. There is
|
||||
# NO NAT hop of any kind, and NO firewall change was needed — verified 2026-09-04 from all
|
||||
# three nodes. The firewalld note further down is a DEV-ERA libvirt concern; it does not
|
||||
# apply to prod, so do not "fix" prod by opening zones.
|
||||
#
|
||||
# DEV (libvirt VMs, HISTORICAL): HOST_NAT_IP=10.73.0.1 was the k3s-dev libvirt NAT gateway.
|
||||
# *** THAT NETWORK IS RETIRED. 10.73.0.1 is stale and must not be reused. *** Dev's own
|
||||
# bootstrap Gitea was later reached on 192.168.2.103:3000 and is retained STOPPED as
|
||||
# recovery (see the comment on the GitRepository patch in clusters/dev/kustomization.yaml).
|
||||
#
|
||||
# WHO RUNS THIS: the OPERATOR, or an agent the operator has EXPLICITLY DIRECTED to stand up a
|
||||
# bootstrap source for a specific bring-up (as happened 2026-09-04 for prod). It is NOT part of
|
||||
# author-only work and must never be fired off casually or "to see if it works": it starts a
|
||||
# long-lived container, generates and writes admin credentials to $HOME, creates an org and repo,
|
||||
# and PUSHES THE CURRENT HEAD to that repo's `main` — which is the revision a cluster pointed at
|
||||
# it will then reconcile. Know which HEAD you are publishing before you run it.
|
||||
set -euo pipefail
|
||||
ENGINE="${ENGINE:-podman}"
|
||||
GITEA_IMAGE="${GITEA_IMAGE:-docker.io/gitea/gitea:1.22}" # pinned; bump if needed
|
||||
@@ -16,11 +35,24 @@ SSH_PORT="${GITEA_SSH_PORT:-2222}"
|
||||
# subuid that can't write a host-owned dir, so Gitea fails to create /data/gitea/conf/app.ini.
|
||||
DATA_VOLUME="${GITEA_DATA_VOLUME:-gitea-bootstrap-data}"
|
||||
DATA_DIR="podman volume ${DATA_VOLUME}" # display label only (data lives in the named volume)
|
||||
# The cluster VMs reach the host on the k3s-dev NAT gateway (10.73.0.1). ROOT_URL must match so
|
||||
# The address the CLUSTER uses to reach this host (see the header). ROOT_URL must match it so
|
||||
# clone/redirect URLs are correct from inside the cluster.
|
||||
# NOTE: firewalld's `libvirt` zone (owns virbrN) blocks VM->host:3000 by default. Open it once:
|
||||
#
|
||||
# The DEFAULT IS DELIBERATELY UNSET — there is no correct default any more. The old default
|
||||
# (10.73.0.1) was the retired k3s-dev libvirt NAT gateway; leaving it in place would silently
|
||||
# publish a ROOT_URL no cluster can resolve, and Flux would report what looks like a network fault.
|
||||
# Pass it explicitly, e.g. for prod: HOST_NAT_IP=192.168.2.132 bash bootstrap-gitea.sh
|
||||
#
|
||||
# NOTE (DEV/libvirt ONLY, historical): firewalld's `libvirt` zone (owns virbrN) blocked
|
||||
# VM->host:3000 by default and had to be opened once with
|
||||
# sudo firewall-cmd --zone=libvirt --add-port=3000/tcp --permanent && sudo firewall-cmd --reload
|
||||
HOST_NAT_IP="${HOST_NAT_IP:-10.73.0.1}"
|
||||
# PROD needs NO firewall change: same-LAN, no libvirt zone involved (verified 2026-09-04).
|
||||
if [ -z "${HOST_NAT_IP:-}" ]; then
|
||||
echo "ERROR: HOST_NAT_IP is not set — it is this host's address AS SEEN FROM THE CLUSTER." >&2
|
||||
echo " prod: HOST_NAT_IP=192.168.2.132 (same LAN as the nodes, no NAT, no firewall change)" >&2
|
||||
echo " There is no safe default: the old one (10.73.0.1) is the RETIRED libvirt NAT." >&2
|
||||
exit 1
|
||||
fi
|
||||
ORG="${GITEA_ORG:-Home-Network}"
|
||||
REPO="${GITEA_REPO:-home-network-k3s-impl}"
|
||||
|
||||
@@ -75,8 +107,17 @@ cat <<EOF
|
||||
|
||||
Gitea bootstrap source ready (${DATA_DIR}).
|
||||
UI (host): ${BASE}/${ORG}/${REPO}
|
||||
Flux GitRepository (cluster): http://${HOST_NAT_IP}:${HTTP_PORT}/${ORG}/${REPO}.git (clusters/dev)
|
||||
Flux GitRepository (cluster): http://${HOST_NAT_IP}:${HTTP_PORT}/${ORG}/${REPO}.git
|
||||
(patch this url into the overlay being bootstrapped:
|
||||
clusters/prod today; clusters/dev has already cut over)
|
||||
Admin creds (host, 0600): ${ADMIN_CRED_FILE}
|
||||
Prerequisite: firewalld must allow VM->host:${HTTP_PORT} —
|
||||
Reachability: prod nodes reach ${HOST_NAT_IP}:${HTTP_PORT} directly on the LAN — no NAT, no
|
||||
firewall change (verified 2026-09-04). For libvirt-based clusters ONLY, firewalld's libvirt zone
|
||||
must allow VM->host:${HTTP_PORT} —
|
||||
ansible-playbook infra/ansible/playbooks/host-prereqs.yml --ask-become-pass
|
||||
REMINDER: the push above published the CURRENT HEAD as this repo's \`main\`. Whatever cluster points
|
||||
at this source will reconcile THAT revision.
|
||||
TEMPORARY BY DESIGN: this container is outside every cluster and is a single point of failure for
|
||||
any cluster reconciling from it. Cut over to in-cluster Gitea (docs/05); for prod the cutover
|
||||
procedure is written on the GitRepository patch in clusters/prod/kustomization.yaml.
|
||||
EOF
|
||||
+5
-5
@@ -24,9 +24,9 @@ env:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN # correct public clone/redirect URLs for the TLS-terminated host
|
||||
value: gitea.home.keithjackson.dev
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.home.keithjackson.dev/
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED # Phase 6 CI/CD: declared (not implicit upstream default) so the
|
||||
value: "true" # in-cluster act_runner (ns gitea-actions) registers against it
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL # resolve `uses: <org>/<action>@ref` from github.com (cluster
|
||||
@@ -104,7 +104,7 @@ lifecycle:
|
||||
# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker
|
||||
# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays
|
||||
# usable). The verify-then-retry loop is the real guarantee against "exit 0 but nothing stuck".
|
||||
GITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
|
||||
GITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration
|
||||
gitea_oidc_fail() {
|
||||
echo "[gitea-oidc] FAILED to register 'keycloak' login source: $1" >&2
|
||||
{ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true
|
||||
@@ -168,7 +168,7 @@ lifecycle:
|
||||
# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,
|
||||
# the moment its OIDC guard was fixed and the block actually ran for the first time.
|
||||
exit 0
|
||||
exposure: public # edge-forwarded git host at gitea.home.keithjackson.dev (docs/09; R-DNS/R-NET)
|
||||
exposure: public # edge-forwarded git host at gitea.${BASE_DOMAIN} (docs/09; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
- name: data
|
||||
@@ -178,7 +178,7 @@ storage:
|
||||
# the default /data/<name> would strand real state on the ephemeral rootfs (R-STATE-01)
|
||||
dependencies: [] # default-deny: reaches only DNS; git clients reach it north-south via the Gateway (R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: gitea.home.keithjackson.dev
|
||||
- host: gitea.${BASE_DOMAIN}
|
||||
port: 3000 # Gitea HTTP web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
|
||||
|
||||
@@ -17,7 +17,7 @@ name: jackett
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/jackett:latest
|
||||
exposure: internal # LAN-only web UI at jackett.home.keithjackson.dev (docs/13 row 4; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
||||
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
- name: config
|
||||
@@ -25,7 +25,7 @@ storage:
|
||||
class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: jackett.home.keithjackson.dev
|
||||
- host: jackett.${BASE_DOMAIN}
|
||||
port: 9117 # Jackett WebUI; protected at the Gateway by forward-auth (docs/09, docs/15)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
@@ -18,7 +18,7 @@ dependencies:
|
||||
port: 5432
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: keycloak.home.keithjackson.dev
|
||||
- host: keycloak.${BASE_DOMAIN}
|
||||
port: 8080 # HTTP listener; TLS terminates at the Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: native # it IS the IdP; own admin login is local + MFA (docs/15; ADR-0014). Documented exception.
|
||||
|
||||
@@ -50,13 +50,13 @@ env:
|
||||
# the vhost answer for the public FQDN; OVERWRITE* make Nextcloud emit https:// self-URLs even
|
||||
# though it speaks plain HTTP behind the proxy. ---
|
||||
- name: NEXTCLOUD_TRUSTED_DOMAINS
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEHOST
|
||||
value: nextcloud.home.keithjackson.dev
|
||||
value: nextcloud.${BASE_DOMAIN}
|
||||
- name: OVERWRITEPROTOCOL
|
||||
value: https
|
||||
- name: OVERWRITECLIURL
|
||||
value: https://nextcloud.home.keithjackson.dev
|
||||
value: https://nextcloud.${BASE_DOMAIN}
|
||||
# --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential
|
||||
# `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret
|
||||
# (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it
|
||||
@@ -88,7 +88,7 @@ lifecycle:
|
||||
# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as
|
||||
# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.
|
||||
OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; }
|
||||
NEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration
|
||||
NEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration
|
||||
nextcloud_oidc_fail() {
|
||||
echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2
|
||||
{ echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true
|
||||
@@ -228,7 +228,7 @@ dependencies:
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
|
||||
ingressRoutes:
|
||||
- host: nextcloud.home.keithjackson.dev
|
||||
- host: nextcloud.${BASE_DOMAIN}
|
||||
port: 80 # apache HTTP; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ storage:
|
||||
class: longhorn # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03)
|
||||
dependencies: [] # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: pihole.home.keithjackson.dev
|
||||
- host: pihole.${BASE_DOMAIN}
|
||||
port: 80 # admin web UI, protected at the Gateway by forward-auth (docs/09)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ storage:
|
||||
# so a placeholder/absent claim Secret would crashloop the pod. See operator hand-off.
|
||||
dependencies: [] # default-deny: reaches only DNS; clients/Seerr reach IT (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: plex.home.keithjackson.dev
|
||||
- host: plex.${BASE_DOMAIN}
|
||||
port: 32400 # Plex web UI + API served on 32400 (wildcard TLS at the Gateway; docs/08/09)
|
||||
auth:
|
||||
mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02)
|
||||
|
||||
@@ -32,7 +32,7 @@ name: qbittorrent
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at qbittorrent.home.keithjackson.dev (docs/13 row 13; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at qbittorrent.${BASE_DOMAIN} (docs/13 row 13; R-DNS/R-NET)
|
||||
egress: vpn # ALL public egress via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
- name: config
|
||||
@@ -40,7 +40,7 @@ storage:
|
||||
class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.home.keithjackson.dev
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
port: 8080 # qBittorrent WebUI; protected at the Gateway by forward-auth (docs/09, docs/15)
|
||||
auth:
|
||||
mode: forward-auth # oauth2-proxy -> Keycloak; bare UI, no usable native OIDC (docs/15; R-IDP-02)
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at radarr.home.keithjackson.dev (docs/13 row 3; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at radarr.${BASE_DOMAIN} (docs/13 row 3; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
@@ -29,7 +29,7 @@ dependencies:
|
||||
port: 9117
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: radarr.home.keithjackson.dev
|
||||
- host: radarr.${BASE_DOMAIN}
|
||||
port: 7878 # Radarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
|
||||
+11
-2
@@ -44,7 +44,16 @@ except ImportError: # pragma: no cover - explicit, actionable failure
|
||||
|
||||
# --- Label / naming conventions (match platform/kyverno/policies + platform/*/) --------
|
||||
LBL = "homelab.keithjackson.dev"
|
||||
ZONE = "home.keithjackson.dev"
|
||||
# The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal.
|
||||
# Every rendered manifest lands under a Flux path whose Kustomization declares
|
||||
# postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at
|
||||
# reconcile time: dev renders dev.home.keithjackson.dev, prod renders home.keithjackson.dev (the
|
||||
# APEX — operator decision 2026-09-04; prod IS the apex, so there is no `prod.` label and no CNAMEs).
|
||||
# It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE —
|
||||
# the escape contract below reserves `$${...}` for expansions meant for a container's shell.
|
||||
# Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
|
||||
# and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10).
|
||||
ZONE = "${BASE_DOMAIN}"
|
||||
GATEWAY_NAME = "shared"
|
||||
GATEWAY_NS = "gateway"
|
||||
KP_RELEASE = "kube-prometheus-stack" # ServiceMonitor/PrometheusRule discovery label
|
||||
@@ -598,7 +607,7 @@ def r_internet_egress(s: dict) -> dict:
|
||||
def r_httproutes(s: dict) -> list:
|
||||
"""One HTTPRoute per ingressRoute, attaching host -> service port on the shared Gateway.
|
||||
|
||||
TLS is the Gateway's wildcard *.home.keithjackson.dev cert (no per-service cert step);
|
||||
TLS is the Gateway's wildcard *.${BASE_DOMAIN} cert (no per-service cert step);
|
||||
ExternalDNS manages the record from the HTTPRoute (R-TLS/R-DNS, renderer-contract).
|
||||
"""
|
||||
out = []
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ dependencies:
|
||||
port: 32400
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: seerr.home.keithjackson.dev
|
||||
- host: seerr.${BASE_DOMAIN}
|
||||
port: 5055 # Overseerr web UI + API; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: native # Plex account auth — documented exception, NOT Keycloak (docs/13/15; R-IDP-02)
|
||||
|
||||
@@ -58,7 +58,7 @@ env:
|
||||
- name: SEMAPHORE_ADMIN_NAME
|
||||
value: Semaphore Admin
|
||||
- name: SEMAPHORE_ADMIN_EMAIL
|
||||
value: admin@home.keithjackson.dev
|
||||
value: admin@${BASE_DOMAIN}
|
||||
# --- Persistent access-key encryption key (see header): FIXED base64 32-byte key from SOPS so the
|
||||
# SSH keys / credentials Semaphore stores in the DB stay decryptable across restarts. ---
|
||||
- name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
|
||||
@@ -110,7 +110,7 @@ dependencies:
|
||||
port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no
|
||||
protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
|
||||
ingressRoutes:
|
||||
- host: semaphore.home.keithjackson.dev
|
||||
- host: semaphore.${BASE_DOMAIN}
|
||||
port: 3000 # Semaphore HTTP web/API; wildcard TLS terminates at the shared Gateway (docs/08/09)
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ name: sonarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
exposure: internal # LAN-only web UI at sonarr.home.keithjackson.dev (docs/13 row 2; R-DNS/R-NET)
|
||||
exposure: internal # LAN-only web UI at sonarr.${BASE_DOMAIN} (docs/13 row 2; R-DNS/R-NET)
|
||||
egress: vpn # indexer scrape + downloader handoff steered via the shared VPN egress gateway,
|
||||
# kill-switch enforced OUTSIDE the workload (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
@@ -29,7 +29,7 @@ dependencies:
|
||||
port: 9117
|
||||
protocol: tcp
|
||||
ingressRoutes:
|
||||
- host: sonarr.home.keithjackson.dev
|
||||
- host: sonarr.${BASE_DOMAIN}
|
||||
port: 8989 # Sonarr web/API; TLS terminates at the shared Gateway (docs/08, docs/09)
|
||||
auth:
|
||||
mode: forward-auth # bare UI, no usable OIDC: oauth2-proxy → Keycloak at the Gateway (docs/15; R-IDP-02)
|
||||
|
||||
@@ -17,15 +17,103 @@ data:
|
||||
CLUSTER_ENV: "base" # dev | prod (overlays set this)
|
||||
|
||||
# --- DNS / TLS ---
|
||||
BASE_DOMAIN: "home.keithjackson.dev" # platform hostname suffix (same both envs; split-horizon)
|
||||
DNS_ZONE: "keithjackson.dev" # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it);
|
||||
# ExternalDNS domain-filter must be the zone, not the subdomain
|
||||
# THE platform hostname suffix. Every public hostname the cluster serves is `<svc>.${BASE_DOMAIN}`,
|
||||
# the wildcard cert is `*.${BASE_DOMAIN}`, coredns-custom answers the `${BASE_DOMAIN}` zone with
|
||||
# ${GATEWAY_VIP}, and every OIDC issuer/redirect URI is built from it.
|
||||
#
|
||||
# THIS KNOB WAS DEAD UNTIL 2026-09-04 — declared here but referenced by NO manifest, because every
|
||||
# hostname was hardcoded. That is what made the 2026-09-04 incident possible: prod and dev were
|
||||
# authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns
|
||||
# (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM.
|
||||
# See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN
|
||||
# suffix, so the two clusters can no longer describe the same name.
|
||||
#
|
||||
# dev -> dev.home.keithjackson.dev (a subdomain; = this base default)
|
||||
# prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label)
|
||||
#
|
||||
# OPERATOR DECISION 2026-09-04 (supersedes the interim split committed earlier the same day, which
|
||||
# had prod on `prod.home.` with apex CNAMEs pointing at it). PROD IS THE APEX. That deletes the
|
||||
# whole CNAME layer — `<svc>.home.keithjackson.dev` is prod's real name, not an alias — and it is
|
||||
# DEV that carries a label. Convention is unchanged from GATEWAY_VIP / LONGHORN_DATA_PATH: the base
|
||||
# default is the DEV value (dev restates it verbatim) and prod overrides it. What changed is which
|
||||
# string dev holds.
|
||||
#
|
||||
# DEV'S VALUE IS A MIGRATION, NOT A VARIABLE CHANGE. The live dev cluster is serving the APEX right
|
||||
# now: its wildcard cert, every public DNS record, its Keycloak issuer, ~12 OIDC clients' redirect
|
||||
# URIs, kubectl SSO, and gitea.home.keithjackson.dev — which IS this repo's GitOps source AND the
|
||||
# push target. Flipping this value on dev without first standing up the new host set STRANDS DEV:
|
||||
# Flux's GitRepository url moves to gitea.dev.home.keithjackson.dev, which has no DNS record and no
|
||||
# cert, so dev stops reconciling and cannot be fixed by pushing. ORDERED CUTOVER, operator present:
|
||||
# infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# DEV MUST VACATE THE APEX BEFORE PROD PUBLISHES THERE. Both clusters describing
|
||||
# `*.home.keithjackson.dev` at once is the 2026-09-04 incident precondition, re-created.
|
||||
BASE_DOMAIN: "dev.home.keithjackson.dev"
|
||||
|
||||
# ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind
|
||||
# EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered,
|
||||
# so it cannot be created, updated or deleted regardless of TXT ownership.
|
||||
#
|
||||
# BOTH ENVIRONMENTS USE THE CLOUDFLARE **ZONE**, `keithjackson.dev`. This value is NOT a per-
|
||||
# environment isolation knob and must not be narrowed to either environment's BASE_DOMAIN.
|
||||
#
|
||||
# FINDING F-DNS1 (DECISIONS-LOG 2026-08-17) — the Cloudflare provider uses --domain-filter for ZONE
|
||||
# SELECTION as well as name filtering, so a filter that is a STRICT SUBDOMAIN of the hosted zone
|
||||
# matches NO zone: ExternalDNS generates the record and then skips it with "no hosted zone matching
|
||||
# record DNS Name", publishing NOTHING. This has been tried and reverted more than once (an agent
|
||||
# narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in review). DO NOT "harden"
|
||||
# it — the change is not defence in depth, it is a total outage of DNS publication.
|
||||
#
|
||||
# WHERE ISOLATION ACTUALLY COMES FROM, and it is sufficient:
|
||||
# (1) the two environments request DISJOINT hostname sets (dev.home.… vs the apex), and
|
||||
# (2) EXTERNALDNS_TXT_OWNER_ID differs, so neither registry ever claims the other's records.
|
||||
# That pairing is what the 2026-09-04 incident needed; a domain filter cannot provide it, because
|
||||
# both environments live in the one zone either way.
|
||||
EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev"
|
||||
|
||||
# ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT:
|
||||
# neither cluster set it, so both inherited the chart default `default`, and ExternalDNS treats a
|
||||
# record whose TXT registry entry carries ITS OWN owner id as its own to reconcile — and with
|
||||
# `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on
|
||||
# one zone therefore garbage-collect each other. Each environment MUST have a distinct value.
|
||||
#
|
||||
# dev -> `dev` prod -> `prod` base default -> a non-colliding sentinel (below)
|
||||
#
|
||||
# DEV IS `dev`, NOT `default` (changed 2026-09-04 with the move to dev.home.keithjackson.dev). Until
|
||||
# this migration dev HAD to render `default`, because its LIVE apex records were already registered
|
||||
# in Cloudflare's TXT registry under that owner and renaming would have orphaned every one of them.
|
||||
# THAT CONSTRAINT NO LONGER BINDS: dev is publishing an ENTIRELY NEW record set under
|
||||
# *.dev.home.keithjackson.dev, with no pre-existing registry entries to recognise, so the owner id
|
||||
# is now free to be chosen on its merits. Two reasons to take `dev`:
|
||||
# - `default` is THE VALUE YOU GET BY FORGETTING (it is the external-dns chart default, and both
|
||||
# clusters silently shared it, which IS the incident). An environment whose identity is the
|
||||
# forget-value can never be told apart from a misconfigured newcomer. No environment should
|
||||
# hold it.
|
||||
# - It makes vacating the apex an EXPLICIT, ORDERED, VERIFIABLE step. Dev's old apex records stay
|
||||
# owned by `default`, which no environment now claims, so nothing deletes them implicitly on
|
||||
# some future reconcile — they are retired deliberately, and prod's publish is gated on that
|
||||
# retirement being confirmed in Cloudflare (infra/PROD-PROVISIONING.md §10) rather than on a
|
||||
# race between two reconcile loops.
|
||||
#
|
||||
# PROD STAYS `prod` AND MUST NOT BECOME `default`, even though prod now serves the apex names that
|
||||
# dev's `default`-owned records currently occupy. Giving prod `default` would hand prod ownership of
|
||||
# the LIVE dev cluster's records while dev is still serving them, and under `policy: sync` prod
|
||||
# would delete the ones no prod source claims. That is the 2026-09-04 incident, re-armed, with the
|
||||
# roles swapped. Prod inherits the apex NAMES; it must never inherit the apex OWNER ID.
|
||||
#
|
||||
# THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default` and not any real
|
||||
# environment's id. A new environment that forgets this knob must not silently inherit another's
|
||||
# identity — that is precisely the incident. With this sentinel such a cluster owns nothing, so it
|
||||
# can DELETE nothing; it would publish under a visibly wrong owner id (grep the TXT records)
|
||||
# instead of destroying a live environment's DNS.
|
||||
EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars"
|
||||
|
||||
ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
|
||||
GATEWAY_VIP: "192.168.1.53" # in-cluster split-horizon target (R-DNS-01/03): the L2-announced
|
||||
# ingress VIP of the `shared` Gateway. MUST equal the FIRST address of
|
||||
# ${CILIUM_LB_POOL_START}..${CILIUM_LB_POOL_STOP} below — the Gateway
|
||||
# takes the pool's first free IP, and coredns-custom answers
|
||||
# *.home.keithjackson.dev with THIS so pod -> public-hostname traffic
|
||||
# *.${BASE_DOMAIN} with THIS so pod -> public-hostname traffic
|
||||
# hairpins through the Gateway (correct SNI -> wildcard cert).
|
||||
# Base default = the DEV LAN VIP (was 10.73.0.240, an address on the
|
||||
# long-RETIRED libvirt network — corrected 2026-08-31 so an overlay that
|
||||
|
||||
@@ -5,6 +5,20 @@
|
||||
# reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02).
|
||||
# It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git
|
||||
# on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1).
|
||||
#
|
||||
# ##############################################################################################
|
||||
# ### *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) *** ###
|
||||
# ### ###
|
||||
# ### This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex ###
|
||||
# ### `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url ###
|
||||
# ### moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a ###
|
||||
# ### merge to main IS the cutover — it executes on the live dev cluster with nobody present. ###
|
||||
# ### Dev's Gitea is both its GitOps source and the push target: if the new host is not up ###
|
||||
# ### with a Ready cert first, dev stops reconciling and pushing cannot repair it. ###
|
||||
# ### ###
|
||||
# ### Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator ###
|
||||
# ### present. Merging is step D3, not step one. ###
|
||||
# ##############################################################################################
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
@@ -131,7 +145,24 @@ patches:
|
||||
# via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO
|
||||
# secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and
|
||||
# revert this url (docs/05).
|
||||
url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
#
|
||||
# *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. ***
|
||||
# It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized —
|
||||
# it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO
|
||||
# postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and
|
||||
# break the source outright (the L0 flux-postbuild-substitution check asserts exactly this).
|
||||
# So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below.
|
||||
#
|
||||
# It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That
|
||||
# makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization
|
||||
# (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE
|
||||
# DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already
|
||||
# resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED
|
||||
# BY PUSHING — the broken thing is the path Flux uses to read pushes.
|
||||
# ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root
|
||||
# Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for
|
||||
# the migration window, and only restore the in-cluster url after the new host is proven.
|
||||
url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
# --- dev-scoped cluster-vars (R-ENV-02) ---
|
||||
@@ -148,6 +179,60 @@ patches:
|
||||
- op: replace
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# ==========================================================================================
|
||||
# DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.**
|
||||
# ==========================================================================================
|
||||
# OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day:
|
||||
#
|
||||
# dev -> dev.home.keithjackson.dev (this knob; = the base default, restated)
|
||||
# prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label,
|
||||
# and therefore no apex CNAME layer at all)
|
||||
#
|
||||
# *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right
|
||||
# now. Merging this file does not perform the migration — it DEFINES the target. The cutover is
|
||||
# an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20
|
||||
# lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps
|
||||
# source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute,
|
||||
# DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does
|
||||
# not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED
|
||||
# BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery
|
||||
# is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair
|
||||
# in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host
|
||||
# FIRST, or park the source on the bootstrap Gitea for the window.
|
||||
#
|
||||
# ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh
|
||||
# ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s
|
||||
# server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's
|
||||
# split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "dev.home.keithjackson.dev"
|
||||
# ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING
|
||||
# F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a
|
||||
# strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching
|
||||
# record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it
|
||||
# newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
value: "keithjackson.dev"
|
||||
# OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex
|
||||
# records were already in Cloudflare's TXT registry under owner `default`, so renaming would
|
||||
# have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix
|
||||
# dev publishes a brand-new record set with no prior registry entries to recognise, so the id
|
||||
# is chosen on its merits, and `default` has none — it is the value external-dns uses when you
|
||||
# FORGET the flag, which is exactly how both clusters came to share it. Consequences, both
|
||||
# deliberate:
|
||||
# - dev's OLD apex records remain owned by `default`, which no environment now claims, so
|
||||
# nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first
|
||||
# publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing.
|
||||
# - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev
|
||||
# cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER.
|
||||
# dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "dev"
|
||||
# --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) -----------------------------
|
||||
# EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it
|
||||
# equals the base default. These values are what the LIVE dev cluster is running right now, so
|
||||
@@ -156,7 +241,7 @@ patches:
|
||||
#
|
||||
# Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN
|
||||
# (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers
|
||||
# *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
|
||||
# *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
|
||||
- op: replace
|
||||
path: /data/GATEWAY_VIP
|
||||
value: "192.168.1.53"
|
||||
|
||||
@@ -115,24 +115,78 @@ patches:
|
||||
name: sops-age
|
||||
# --- GitOps source (operator decision 2026-09-01: prod tracks the SAME BRANCH as dev) ---
|
||||
#
|
||||
# *** BOOTSTRAP BLOCKER — this URL CANNOT work on a cold prod cluster. READ BEFORE BRING-UP. ***
|
||||
# ############################################################################################
|
||||
# ### ###
|
||||
# ### *** T E M P O R A R Y B O O T S T R A P S O U R C E — NOT PERMANENT *** ###
|
||||
# ### ###
|
||||
# ### The url below is a Gitea container running ON THE OPERATOR'S WORKSTATION, outside ###
|
||||
# ### any cluster. It exists ONLY to get prod's first reconcile off the ground. It MUST ###
|
||||
# ### be cut over to the in-cluster URL once Gitea runs on prod. If you are reading this ###
|
||||
# ### and prod has been serving traffic for months, THE CUTOVER WAS NEVER DONE — that is ###
|
||||
# ### a live single point of failure, not a settled design. See "CUTOVER" below. ###
|
||||
# ### ###
|
||||
# ############################################################################################
|
||||
#
|
||||
# Two independent reasons, both consequences of the split-horizon design:
|
||||
# 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's
|
||||
# GitOps source lives inside the very cluster slated for decommission. Gitea must move to
|
||||
# prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth.
|
||||
# 2. SPLIT-HORIZON DNS SENDS IT TO THE WRONG PLACE. platform/coredns/coredns-custom.yaml
|
||||
# answers the ENTIRE home.keithjackson.dev zone with ${GATEWAY_VIP}. On prod that is
|
||||
# 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve
|
||||
# gitea.home.keithjackson.dev to itself and never reach a repository.
|
||||
# WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the
|
||||
# unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`).
|
||||
# REASONING UPDATED 2026-09-04 for the new suffix direction (prod = the apex, dev = dev.home.),
|
||||
# which changes WHY the blocker holds but not THAT it holds:
|
||||
# 1. SELF-HOSTING CIRCULARITY — still true, and now the only reason. The Gitea this repo is
|
||||
# pushed to runs ON THE DEV CLUSTER. Prod syncing from it would put prod's GitOps source
|
||||
# inside the very cluster slated for decommission. Gitea must move to prod (or off-cluster)
|
||||
# BEFORE dev is retired, or prod loses its source of truth.
|
||||
# NOTE the suffix change makes the two Giteas DIFFERENT HOSTS: dev's is now
|
||||
# gitea.dev.home.keithjackson.dev and prod's is gitea.home.keithjackson.dev. Prod pointing at
|
||||
# the apex name therefore means "prod's OWN Gitea", never dev's — which is what makes the
|
||||
# cutover below a one-line change once prod actually runs Gitea.
|
||||
# 2. SPLIT-HORIZON DNS — NO LONGER A BLOCKER, and the note is kept because the old reasoning is
|
||||
# easy to re-derive incorrectly. platform/coredns/coredns-custom.yaml answers the whole
|
||||
# ${BASE_DOMAIN} zone with ${GATEWAY_VIP}; when prod's BASE_DOMAIN was `prod.home.` that meant
|
||||
# prod resolved gitea.home.keithjackson.dev to PROD's gateway (192.168.2.20) where no Gitea
|
||||
# listened — a misdirection. Now that prod IS the apex, resolving gitea.home.keithjackson.dev
|
||||
# to prod's own Gateway is CORRECT: that is where prod's Gitea will be served from. What
|
||||
# remains is purely reason 1 — the Gitea is not there YET.
|
||||
#
|
||||
# Bring-up therefore needs a TEMPORARY source (the bootstrap host-Gitea, or an off-cluster
|
||||
# mirror), with a cutover to the in-cluster URL once Gitea actually runs on prod — exactly the
|
||||
# sequence docs/05 describes for dev's own Phase-6 cutover. Tracked in PROD-PROVISIONING.md §7.
|
||||
# WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction):
|
||||
# bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the
|
||||
# operator's workstation, Gitea 1.22.6, listening on 192.168.2.132:3000. All three prod nodes
|
||||
# reach it DIRECTLY on the LAN: no libvirt NAT hop (that network is retired) and no firewall
|
||||
# change was needed. This is the same "local bootstrap source" pattern dev itself used before
|
||||
# its Phase-6 cutover (docs/05 "Source-of-truth").
|
||||
#
|
||||
# PLAIN HTTP, AND WHY THAT IS ACCEPTED HERE: the bootstrap Gitea has no TLS, so there is no cert
|
||||
# for Flux to trust and no caCertSecretRef to wire. Traffic is LAN-local between prod nodes and
|
||||
# the workstation, the repo carries NO plaintext secrets (everything sensitive is SOPS-encrypted,
|
||||
# .sops.yaml), and the window is the bring-up only. Do NOT extend this posture past cutover.
|
||||
#
|
||||
# NO secretRef, DELIBERATELY: the repo is PUBLIC on the bootstrap Gitea, so anonymous clone
|
||||
# works and Flux needs no git credentials. VERIFIED 2026-09-04 by unauthenticated request:
|
||||
# curl http://192.168.2.132:3000/api/v1/repos/Home-Network/home-network-k3s-impl -> "private":false
|
||||
# git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git -> refs/heads/main
|
||||
# If the GitRepository ever reports an auth failure that looks like a network problem, RE-CHECK
|
||||
# THAT FIRST: a private repo here fails exactly that way.
|
||||
#
|
||||
# *** CUTOVER — the exit condition, mirroring dev's Phase-6 cutover (docs/05) ***
|
||||
# WHEN: Gitea runs on prod (reconciled in-cluster) and answers gitea.home.keithjackson.dev
|
||||
# through prod's own Gateway VIP with a publicly-trusted cert.
|
||||
# DO: 1. push/mirror this repo into the prod in-cluster Gitea;
|
||||
# 2. change the url below to
|
||||
# https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
# and DELETE this whole TEMPORARY block (dev's equivalent stanza, ~15 lines above in
|
||||
# clusters/dev/kustomization.yaml, is what "done" looks like);
|
||||
# 3. confirm `flux get sources git -n flux-system` reconciles the new revision;
|
||||
# 4. only THEN stop the workstation container. Keep it stopped-not-deleted as recovery,
|
||||
# exactly as dev retained its bootstrap Gitea.
|
||||
# UNTIL THEN: prod's reconciliation depends on a container on a workstation that may be
|
||||
# powered off. Flux keeps the last-reconciled state running, but NO new revision lands while
|
||||
# it is down. Tracked as an open blocker in infra/PROD-PROVISIONING.md §7.
|
||||
#
|
||||
# The BRANCH below is settled and correct: prod tracks `main`, the same branch as dev.
|
||||
# CONSEQUENCE, recorded deliberately: once prod is live, EVERY MERGE TO main RECONCILES BOTH
|
||||
# CLUSTERS. There is no promotion step and no soak window — dev and prod move together.
|
||||
# NOTE for bring-up: bootstrap-gitea.sh pushes the operator's CURRENT HEAD to the bootstrap
|
||||
# Gitea's `main`, whatever branch that is. Before bootstrapping prod, confirm that remote `main`
|
||||
# is the revision you intend prod to run (`git ls-remote` above, against local `main`).
|
||||
- patch: |-
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: GitRepository
|
||||
@@ -140,7 +194,9 @@ patches:
|
||||
name: flux-system
|
||||
namespace: flux-system
|
||||
spec:
|
||||
url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
|
||||
# TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block
|
||||
# above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final.
|
||||
url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
- target:
|
||||
@@ -154,6 +210,79 @@ patches:
|
||||
- op: replace
|
||||
path: /data/ACME_CLUSTER_ISSUER
|
||||
value: "letsencrypt-prod"
|
||||
# ==========================================================================================
|
||||
# PROD DNS IDENTITY (R-DNS-01/02/03) — PROD **IS** THE APEX.
|
||||
# ==========================================================================================
|
||||
# OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day
|
||||
# (prod on `prod.home.` behind apex CNAMEs). The simpler target, and the one authored here:
|
||||
#
|
||||
# dev -> dev.home.keithjackson.dev (dev carries the label; clusters/dev)
|
||||
# prod -> home.keithjackson.dev (THE APEX — this knob)
|
||||
#
|
||||
# There is NO `prod.` label and NO CNAME layer. `<svc>.home.keithjackson.dev` is prod's REAL
|
||||
# name, so nothing has to alias anything: prod's Gitea is gitea.home.keithjackson.dev, its
|
||||
# Keycloak keycloak.home.keithjackson.dev, and the wildcard cert is `*.home.keithjackson.dev`.
|
||||
#
|
||||
# BACKGROUND — WHY THE SUFFIXES HAD TO SPLIT AT ALL. Until 2026-09-04 every hostname was
|
||||
# HARDCODED to home.keithjackson.dev in the base manifests, so prod was authored to serve the
|
||||
# EXACT SAME public names as the live dev cluster out of the SAME Cloudflare zone. Two
|
||||
# consequences, one of which already happened:
|
||||
# 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit
|
||||
# default dev had) concluded it owned dev's records and DELETED EVERY ONE of them.
|
||||
# 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev`
|
||||
# wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would
|
||||
# have raced each other's `_acme-challenge` TXT records on that one zone.
|
||||
#
|
||||
# *** THE ORDERING CONSTRAINT THIS VALUE CREATES — READ BEFORE UNSUSPENDING ANYTHING. ***
|
||||
# Prod now claims the very names the LIVE dev cluster is serving today. Both hazards above are
|
||||
# therefore still armed until DEV HAS VACATED THE APEX. Prod's `platform` and its 10 downstream
|
||||
# tiers are SUSPENDED for exactly this reason. The gate is not "review is done", it is:
|
||||
# (a) dev is reconciling on *.dev.home.keithjackson.dev with a Ready wildcard cert, AND
|
||||
# (b) dev's old apex A/CNAME records and their TXT registry entries are GONE from Cloudflare
|
||||
# (verified against Cloudflare directly — a pfSense override is NOT evidence about public
|
||||
# DNS; that mistake was made on 2026-09-04), AND
|
||||
# (c) dev's old `*.home.keithjackson.dev` cert/Certificate is released, because prod ordering
|
||||
# that same name set while dev's still exists counts against LE's duplicate-certificate
|
||||
# limit (5 per week per identical name set).
|
||||
# Full ordered runbook: infra/PROD-PROVISIONING.md §10. Unsuspending before (a)-(c) re-creates
|
||||
# the 2026-09-04 incident with the roles reversed.
|
||||
#
|
||||
# [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
|
||||
# The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
|
||||
# set Ansible-side from that variable and cannot read this one. If they disagree, kubectl SSO
|
||||
# trusts a different Keycloak than the one prod actually runs.
|
||||
- op: replace
|
||||
path: /data/BASE_DOMAIN
|
||||
value: "home.keithjackson.dev"
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_DOMAIN_FILTER
|
||||
# MUST be the Cloudflare ZONE. Prod now serves the apex, so `home.keithjackson.dev` looks
|
||||
# like a tighter, more honest filter than the zone — IT IS NOT, IT IS AN OUTAGE. FINDING
|
||||
# F-DNS1 (DECISIONS-LOG 2026-08-17) proved a STRICT SUBDOMAIN of the hosted zone drops the
|
||||
# zone entirely: ExternalDNS generates the record then skips it with "no hosted zone matching
|
||||
# record DNS Name", so prod publishes NOTHING. An agent narrowed this to
|
||||
# prod.home.keithjackson.dev on 2026-09-04 and it was reverted in review; do not retry it.
|
||||
# Isolation from dev does NOT come from this filter — it comes from
|
||||
# (1) the two clusters requesting DISJOINT hostname sets (apex vs dev.home.), and
|
||||
# (2) txtOwnerId: prod vs dev, so neither registry ever claims the other's records.
|
||||
# That pairing is what the 2026-09-04 incident needed and this filter cannot provide.
|
||||
value: "keithjackson.dev"
|
||||
# SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so
|
||||
# both inherited the chart default `default`; prod then treated dev's TXT-registry entries as
|
||||
# its own and, under policy: sync, deleted the records behind them.
|
||||
#
|
||||
# PROD MUST NOT BE GIVEN `default`, however tempting it now looks. Prod is taking over the apex
|
||||
# NAMES, and dev's existing apex records are registered under owner `default` — so `default` is
|
||||
# the one value that would let prod adopt them. It would also let prod SYNC-DELETE the ones no
|
||||
# prod source claims, on a cluster dev is still serving. Prod inherits the apex names; it must
|
||||
# never inherit the apex OWNER ID. `prod` keeps prod fail-safe: records owned by `default` are
|
||||
# invisible to it, so an early unsuspend cannot destroy dev's DNS the way 2026-09-04 did — it
|
||||
# simply fails to publish until those records are retired, which is the correct behaviour.
|
||||
# dev is `dev` (clusters/dev, changed with dev's migration); the base default is a
|
||||
# non-colliding sentinel. All three differ.
|
||||
- op: replace
|
||||
path: /data/EXTERNALDNS_TXT_OWNER_ID
|
||||
value: "prod"
|
||||
- op: replace
|
||||
path: /data/LONGHORN_DEFAULT_REPLICA_COUNT
|
||||
value: "3"
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8080
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -76,8 +76,8 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://paas-demo.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
+3
-3
@@ -91,7 +91,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -99,8 +99,8 @@ spec:
|
||||
- --allowed-group=media-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
@@ -91,7 +91,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
- --upstream=http://127.0.0.1:8989
|
||||
- --email-domain=*
|
||||
- --scope=openid email profile
|
||||
@@ -99,8 +99,8 @@ spec:
|
||||
- --allowed-group=media-admins
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
+381
-9
@@ -1172,6 +1172,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
| `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address |
|
||||
| `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target |
|
||||
| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real. Prod's value was inverted the same day to the APEX `home.keithjackson.dev`, with dev moving to `dev.home.keithjackson.dev` — see §10a, which is authored but NOT yet applied. |
|
||||
|
||||
- [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both*
|
||||
overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the
|
||||
@@ -1193,12 +1194,15 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
as dev**. Eight `decryption: {provider: sops, secretRef: sops-age}` patches mirror dev
|
||||
exactly. The `sops-age` Secret is still created OUT OF BAND on the prod cluster
|
||||
(`bootstrap/sops/install-age-key.sh`) — the overlay references it by name only.
|
||||
- [~] **GitRepository source** — BRANCH settled, URL still blocked. Operator decision
|
||||
2026-09-01: **prod tracks `main`, the same branch as dev**, because dev is temporary and
|
||||
will be decommissioned once prod is stable. Recorded consequence: once prod is live,
|
||||
**every merge to `main` reconciles BOTH clusters** — no promotion step, no soak window.
|
||||
The URL is patched in but **cannot work on a cold prod cluster**, for two independent
|
||||
reasons found 2026-09-01:
|
||||
- [~] **GitRepository source** — **MITIGATED 2026-09-04, NOT SOLVED.** Branch settled; the
|
||||
in-cluster URL is still unusable, so prod now points at a **temporary bootstrap source**.
|
||||
Operator decision 2026-09-01: **prod tracks `main`, the same branch as dev**, because dev
|
||||
is temporary and will be decommissioned once prod is stable. Recorded consequence: once
|
||||
prod is live, **every merge to `main` reconciles BOTH clusters** — no promotion step, no
|
||||
soak window.
|
||||
|
||||
**Why the in-cluster URL (`https://gitea.home.keithjackson.dev/...`) cannot work on a
|
||||
cold prod cluster** — two independent reasons, found 2026-09-01, both still true:
|
||||
1. **Self-hosting circularity** — Gitea runs on the DEV cluster, the one slated for
|
||||
decommission. Gitea must move to prod (or off-cluster) BEFORE dev is retired, or prod
|
||||
loses its source of truth. **This is the decommission plan's first task, not its last.**
|
||||
@@ -1206,9 +1210,56 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
|
||||
`home.keithjackson.dev` zone with `${GATEWAY_VIP}`, which on prod is `192.168.2.20`:
|
||||
prod's OWN gateway, where no Gitea listens. Prod's Flux would resolve
|
||||
`gitea.home.keithjackson.dev` to itself and never reach a repository.
|
||||
Bring-up needs a TEMPORARY source (bootstrap host-Gitea, or an off-cluster mirror) and a
|
||||
cutover once Gitea runs on prod — the same sequence `docs/05` describes for dev's own
|
||||
Phase-6 cutover.
|
||||
|
||||
**The mitigation now in `clusters/prod/kustomization.yaml`** (operator directed this to be
|
||||
stood up, 2026-09-04):
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| GitRepository url | `http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` |
|
||||
| Branch | `main` (unchanged — same branch as dev) |
|
||||
| What it is | Gitea 1.22.6 in a **podman container on the operator's workstation**, started by `bootstrap/gitea/bootstrap-gitea.sh` with `HOST_NAT_IP=192.168.2.132` |
|
||||
| Reachability | All three prod nodes reach it **directly on the LAN** — no NAT hop (the libvirt network is retired), **no firewall change needed**. Verified from all three nodes. |
|
||||
| Auth | **None, deliberately.** The repo is PUBLIC on that Gitea, so Flux needs no `secretRef`. Verified anonymously: `curl .../api/v1/repos/Home-Network/home-network-k3s-impl` → `"private":false`, and `git ls-remote` returns `refs/heads/main`. |
|
||||
| Transport | **Plain HTTP, no TLS.** Accepted for the bring-up window only: LAN-local, and the repo holds no plaintext secrets (all SOPS-encrypted). |
|
||||
| Admin creds | `~/.config/home-k3s-bootstrap-gitea-admin.txt`, mode 0600, host-only, never committed. |
|
||||
|
||||
**THIS IS A SINGLE POINT OF FAILURE FOR PROD'S RECONCILIATION, and it is not even in a
|
||||
cluster.** It is one container on one workstation, with no HA, no backup of its own, and
|
||||
no restart guarantee beyond `--restart unless-stopped`. If that workstation is powered
|
||||
off, suspended, or the podman volume `gitea-bootstrap-data` is lost:
|
||||
- already-reconciled state **keeps running** — Flux does not tear anything down;
|
||||
- but **no new revision can land on prod**, and every `GitRepository` reconcile fails.
|
||||
So prod is *not* self-sufficient while this stands. Do not treat prod as production-ready
|
||||
on this footing, and do not let it become the steady state by inattention.
|
||||
|
||||
**One bring-up footgun:** `bootstrap-gitea.sh` pushes the operator's **current `HEAD`** to
|
||||
the bootstrap Gitea's `main`, whatever branch that HEAD is on. Before bootstrapping prod,
|
||||
confirm that remote `main` is the revision prod should run
|
||||
(`git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` vs local
|
||||
`main`). As of 2026-09-04 the remote `main` is the tip of the working branch, **not** local
|
||||
`main`.
|
||||
|
||||
**EXIT CONDITION (the cutover) — mirrors dev's own Phase-6 cutover, `docs/05`.**
|
||||
This item is not closable until all of it is done:
|
||||
- [ ] Gitea runs **on prod**, reconciled in-cluster, answering
|
||||
`gitea.home.keithjackson.dev` through prod's Gateway VIP with a publicly-trusted cert.
|
||||
- [ ] This repo is pushed/mirrored into that in-cluster Gitea.
|
||||
- [ ] The `clusters/prod` GitRepository url is changed to
|
||||
`https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`
|
||||
and the TEMPORARY comment block is **deleted** (dev's stanza is what "done" looks like).
|
||||
- [ ] `flux get sources git -n flux-system` on prod reconciles a revision from the new URL.
|
||||
- [ ] **Only then** stop the workstation container — kept stopped-not-deleted as recovery,
|
||||
exactly as dev retained its own bootstrap Gitea.
|
||||
- [x] **`roles/flux_bootstrap` could only bootstrap dev** — FIXED 2026-09-04. It invoked
|
||||
`bootstrap/flux/bootstrap.sh` without ever exporting `FLUX_ENV_PATH`, so the script fell
|
||||
through to its `clusters/dev` default and §8 step 4 (`--path=clusters/prod`) was
|
||||
unreachable without editing the role. Now parameterised on the standard group_vars seam:
|
||||
role default `flux_bootstrap_env_path: clusters/dev` (defaults/main.yml) is exported as
|
||||
`FLUX_ENV_PATH` by the role's task, and
|
||||
`inventory/group_vars/k3s_prod/vars.yml` overrides it to `clusters/prod` — the same
|
||||
mechanism as `kube_vip_address` / `k3s_data_dir` / `cluster_kubeconfig_name`. Dev has no
|
||||
override anywhere, so dev resolves `clusters/dev` exactly as before.
|
||||
- [ ] Confirm prod's backup target: dev's S3 (`versitygw` on the Proxmox host, `192.168.1.9`) is on
|
||||
the dev side of the network. Off-cluster backup is DR-critical (R-STATE-03).
|
||||
|
||||
@@ -1232,3 +1283,324 @@ single-server-etcd window, and avoids a later member-add dance.
|
||||
specifically.
|
||||
- [ ] Monitoring and alerts live on prod.
|
||||
- [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03).
|
||||
Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10a
|
||||
"The L2 tests hardcode the apex".
|
||||
- [ ] **Prod serves the apex DNS/TLS identity** and dev's records are provably untouched (§10/§10a):
|
||||
distinct `EXTERNALDNS_TXT_OWNER_ID` (`prod` vs `dev`), prod wildcard cert
|
||||
`*.home.keithjackson.dev`, dev migrated to `*.dev.home.keithjackson.dev` with its old apex
|
||||
records retired (§10a step D8), and prod's own Keycloak realm + per-service client secrets.
|
||||
|
||||
## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture**
|
||||
|
||||
> **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's
|
||||
> `platform` tier and its 10 downstream tiers are SUSPENDED. Dev is **live on the apex**. The
|
||||
> ordered cutover is **§10a**, and merging the authored change to `main` is step **D3** of it,
|
||||
> not step one — dev self-reconciles `main`, so a merge outside the runbook window executes
|
||||
> the migration unattended.
|
||||
|
||||
### What happened
|
||||
|
||||
Prod's `external-dns` **deleted every one of dev's public DNS records.**
|
||||
|
||||
Both clusters ran `policy: sync` against the **same Cloudflare zone**, served the **same
|
||||
hostnames**, and — because neither overlay set `txtOwnerId` — both inherited the external-dns
|
||||
chart's default owner id **`default`**. ExternalDNS decides what it may delete by reading its own
|
||||
owner id back out of the TXT registry. Prod read dev's TXT entries, saw its own owner id, found no
|
||||
local resource claiming those names, and garbage-collected the records behind them. Dev recovered
|
||||
once prod was stopped. Prod's `platform` tier and everything downstream were then **suspended**, and
|
||||
remain suspended, so it cannot recur before this change is reviewed.
|
||||
|
||||
`policy: sync` was not the bug and has not been changed. What was missing was **identity**.
|
||||
|
||||
### The second bug, queued directly behind it
|
||||
|
||||
`platform-config` would have requested a `*.home.keithjackson.dev` wildcard `Certificate` from
|
||||
**letsencrypt-prod** — for a cert **dev already holds** — and both clusters would then have raced
|
||||
each other's `_acme-challenge` TXT records on that one zone. Neither would reliably solve DNS-01.
|
||||
This never fired only because the tier was suspended first.
|
||||
|
||||
### Root cause, stated precisely
|
||||
|
||||
Not external-dns. **Every hostname in the repo was a hardcoded literal.** `BASE_DOMAIN` existed in
|
||||
`clusters/base/cluster-vars.yaml` but was referenced by **no reconciled manifest** — the repo-derived
|
||||
status report flagged it as a dead knob. R-ENV-02 promises dev and prod differ only by `cluster-vars`
|
||||
values; for DNS and TLS that promise was false, so `clusters/prod` was *authored* to serve dev's
|
||||
public identity. Two clusters that describe the same DNS name will fight over it no matter how the
|
||||
DNS controller is configured.
|
||||
|
||||
### The architecture decision (operator) — **INVERTED 2026-09-04, this is the current target**
|
||||
|
||||
```
|
||||
dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev
|
||||
prod -> *.home.keithjackson.dev gitea.home.keithjackson.dev (THE APEX)
|
||||
```
|
||||
|
||||
**Prod *is* the apex. There is no `prod.` label and no CNAME layer at all** —
|
||||
`<svc>.home.keithjackson.dev` is prod's real name, not an alias for one. This **supersedes** the
|
||||
interim split committed earlier the same day (`c8c1d29`), which put prod on
|
||||
`*.prod.home.keithjackson.dev` behind apex CNAMEs pointing at it. That layer is deleted: fewer
|
||||
moving parts, no CNAME chain to keep in step, and prod ends up on the names users already type.
|
||||
|
||||
The cost of the inversion is that **dev must move**, and dev is the live cluster. The interim split
|
||||
was chosen precisely because it moved only the unbuilt environment; this one does not have that
|
||||
property, so the whole of §10's remaining risk now sits on dev's side, and it is sequenced below.
|
||||
|
||||
### The values, per environment
|
||||
|
||||
| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | Ansible `k3s_base_domain` |
|
||||
|---|---|---|---|---|
|
||||
| base default | `dev.home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | `dev.home.keithjackson.dev` (role default) |
|
||||
| **dev** | `dev.home.keithjackson.dev` | `dev` | `keithjackson.dev` | (inherits the role default) |
|
||||
| **prod** | `home.keithjackson.dev` | `prod` | `keithjackson.dev` | `home.keithjackson.dev` (`group_vars/k3s_prod`) |
|
||||
|
||||
`[parity]`: `k3s_base_domain` must equal the same cluster's `BASE_DOMAIN`. The kube-apiserver's
|
||||
`oidc-issuer-url` is a k3s **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux
|
||||
post-build substitution can never reach it — hence one fact in two places. If they disagree the
|
||||
apiserver validates tokens against a different Keycloak than the cluster runs: kubectl SSO fails
|
||||
while the cert-based admin kubeconfig keeps working, so it presents as "SSO is broken", not as skew.
|
||||
|
||||
**`EXTERNALDNS_DOMAIN_FILTER` is the Cloudflare ZONE for BOTH, and must stay that way.** FINDING
|
||||
F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for **zone selection** as
|
||||
well as record matching, so a filter that is a strict subdomain of the hosted zone matches **no
|
||||
zone** — external-dns generates the record and then skips it with *"no hosted zone matching record
|
||||
DNS Name"*, publishing **nothing**. This has been "hardened" and reverted more than once. It is not
|
||||
defence in depth; it is a total outage of DNS publication. Isolation comes from the two guards below,
|
||||
which a domain filter cannot provide anyway: both environments live in the one zone regardless.
|
||||
|
||||
**The two guards that actually work:**
|
||||
|
||||
1. **`txtOwnerId`, per-environment identity** — a cluster only ever reconciles records its own owner
|
||||
id claims in the TXT registry. This is the direct fix for the incident.
|
||||
2. **Disjoint hostname sets** — after the migration the two clusters never describe the same record,
|
||||
so there is nothing to fight over. *This guard does not exist during the migration window*, which
|
||||
is the entire reason the ordering below is mandatory.
|
||||
|
||||
#### Why dev's owner id changed from `default` to `dev`
|
||||
|
||||
Until this migration dev **had** to render `default`: its live apex records were already registered
|
||||
in Cloudflare's TXT registry under that owner, and renaming would have orphaned every one of them.
|
||||
**The migration dissolves that constraint** — on `*.dev.home.keithjackson.dev` dev publishes an
|
||||
entirely new record set with no prior registry entries to recognise, so the id is free and is chosen
|
||||
on its merits. `default` has none: it is the value external-dns uses when you **forget** the flag,
|
||||
which is exactly how both clusters came to share it, and an environment whose identity is the
|
||||
forget-value can never be told apart from a misconfigured newcomer. Consequences, both deliberate:
|
||||
|
||||
- Dev's **old apex records stay owned by `default`**, which no environment now claims, so nothing
|
||||
deletes them implicitly on some future reconcile. Retiring them is an explicit step (**D7**) and
|
||||
prod's first publish is gated on that retirement being *confirmed* — not on two reconcile loops
|
||||
racing each other.
|
||||
- **Prod stays `prod` and must never be given `default`.** Prod is taking over the apex *names*, and
|
||||
dev's existing apex records are registered under `default` — so `default` is the one value that
|
||||
would let prod adopt them, and under `policy: sync` delete the ones no prod source claims, on a
|
||||
cluster dev is still serving. That is the 2026-09-04 incident re-armed with the roles swapped.
|
||||
Prod inherits the apex **names**; it must never inherit the apex **owner id**.
|
||||
|
||||
All three values differ, and the base default remains a sentinel so an environment that forgets the
|
||||
knob owns nothing and can therefore delete nothing.
|
||||
|
||||
### PHASE 1 — prod tokenized and moved off dev's names (`c8c1d29`, done; suffix since inverted)
|
||||
|
||||
`BASE_DOMAIN` was a **dead knob** — declared in `clusters/base/cluster-vars.yaml` and referenced by
|
||||
no reconciled manifest, because every hostname was a hardcoded literal. `c8c1d29` made it real (141
|
||||
references across the reconciled set) and gave prod its own suffix. That work stands; only the
|
||||
*value* prod holds has changed, from `prod.home.keithjackson.dev` to the apex.
|
||||
|
||||
### What could NOT be tokenized, and why
|
||||
|
||||
- **`OIDC_ISSUER_URL` / `OIDC_CALLBACK_URL` in `platform/headlamp/headlamp-oidc-credentials.sops.yaml`**
|
||||
— SOPS-encrypted (`encrypted_regex: ^(data|stringData)$`), so the two hostnames are ciphertext.
|
||||
Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and
|
||||
**cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake
|
||||
would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately.
|
||||
**Both environments now need action:** dev's copy must move to `keycloak.dev.home.keithjackson.dev`
|
||||
as part of step **D5**, and prod needs its own copy with `keycloak.home.keithjackson.dev` and a
|
||||
client secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client
|
||||
secret is realm-scoped.
|
||||
- **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s
|
||||
server flag, not a manifest. Handled on the **Ansible** seam instead: role default
|
||||
`k3s_base_domain` (dev's value) + derived `k3s_oidc_issuer_url`, overridden for prod in
|
||||
`inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`,
|
||||
so a fourth prod server inherits it from group membership. Verified per host by a read-only probe
|
||||
(`connection: local`, zero SSH, `changed=0`): all three dev servers resolve
|
||||
`https://keycloak.dev.home.keithjackson.dev/realms/homelab`, all three prod servers resolve
|
||||
`https://keycloak.home.keithjackson.dev/realms/homelab`.
|
||||
- **`spec.url` of dev's `GitRepository`** (`clusters/dev/kustomization.yaml`) — see the strand below.
|
||||
It *cannot* be tokenized: the `flux-system` root Kustomization that renders `./clusters/dev`
|
||||
declares no `postBuild.substituteFrom`, so `gitea.${BASE_DOMAIN}` there would ship the **literal
|
||||
token** and break the source outright (the L0 `flux-postbuild-substitution` check asserts exactly
|
||||
this). It is a hand-maintained literal that must move in lockstep with `BASE_DOMAIN`.
|
||||
|
||||
---
|
||||
|
||||
## 10a. THE MIGRATION RUNBOOK — ordering is load-bearing
|
||||
|
||||
> **Prod's `platform` tier and its 10 downstream tiers are SUSPENDED and must stay suspended until
|
||||
> step P1.** Nothing below is safe to run out of order, and nothing below should be run while the
|
||||
> operator is away.
|
||||
|
||||
### The two ways this goes wrong
|
||||
|
||||
**(1) BOTH CLUSTERS ON THE APEX — the 2026-09-04 incident, recreated.** If prod publishes
|
||||
`*.home.keithjackson.dev` before dev has vacated it, two clusters describe the same records in the
|
||||
same zone again. Distinct `txtOwnerId`s stop prod from *deleting* dev's records — that guard holds —
|
||||
but they do not stop the second failure mode: both clusters requesting a `letsencrypt-prod` wildcard
|
||||
for the **same name set** and racing each other's `_acme-challenge` TXT records, so neither reliably
|
||||
solves DNS-01. **Rule: dev vacates the apex first, and the vacate is verified against Cloudflare
|
||||
before prod is unsuspended.**
|
||||
|
||||
**(2) THE STRAND — dev's GitOps source is the thing that moves.** `spec.url` of dev's
|
||||
`GitRepository` is `https://gitea.home.keithjackson.dev/...`, which is **both** dev's GitOps source
|
||||
**and** the PM's push target. Flipping dev's `BASE_DOMAIN` moves Gitea's HTTPRoute, its public DNS
|
||||
record and the wildcard cert covering it to `gitea.dev.home.keithjackson.dev` all at once. If that
|
||||
host does not already resolve and serve a valid, Ready cert at that instant, **dev stops reconciling
|
||||
and cannot be fixed by pushing** — the broken thing *is* the path Flux uses to read pushes. Recovery
|
||||
is then hands-on and needs LAN access to the cluster.
|
||||
|
||||
**(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root
|
||||
Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main`
|
||||
too. **A merge to `main` therefore *executes* this migration on the live dev cluster, unattended.**
|
||||
The authored change is a *target*, not a deployment. **Do not merge these files to `main` outside the
|
||||
runbook window.** Park them on a branch until the operator is at the keyboard and step **D1** has
|
||||
been done.
|
||||
|
||||
### D — dev vacates the apex (operator present, in this order)
|
||||
|
||||
- [ ] **D0. Snapshot the escape routes before touching anything.** Confirm a working break-glass
|
||||
**cert-based** admin kubeconfig for dev (not the OIDC one — OIDC breaks in D6). Confirm the
|
||||
retained bootstrap Gitea can be started (`bootstrap/gitea/bootstrap-gitea.sh`; dev's historical
|
||||
recovery host was `192.168.2.103:3000`, prod's current bootstrap is `192.168.2.132:3000`).
|
||||
Record dev's current Cloudflare record set (names + TXT registry owners) so D7 has a checklist
|
||||
and so an accidental deletion is detectable.
|
||||
- [ ] **D1. Break the automatic trigger.** Either `flux suspend kustomization flux-system -n
|
||||
flux-system` on dev (stops the root from self-applying the merge), **or** re-point dev's
|
||||
`GitRepository` url at the bootstrap Gitea for the window. Do this **before** the merge, not
|
||||
after. This is what converts "merging strands dev" into "merging is inert until you say go".
|
||||
- [ ] **D2. Publish and prove the new host set *while dev still serves the apex*.** The new names
|
||||
must exist before anything depends on them:
|
||||
- a `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev` **wildcard Certificate issued
|
||||
and `Ready`** (LE prod, DNS-01) — see the ACME note below before ordering it;
|
||||
- public DNS for `gitea.dev.home.keithjackson.dev` resolving to dev's Gateway VIP
|
||||
(192.168.1.53), **checked against Cloudflare / a public resolver, never through the pfSense
|
||||
override** — that mistake was made on 2026-09-04 and produced a false healthy reading;
|
||||
- `curl -sS https://gitea.dev.home.keithjackson.dev/api/v1/version` succeeding with a
|
||||
publicly-trusted cert from a host on the dev LAN.
|
||||
Until D2 is green, D3 is not safe.
|
||||
- [ ] **D3. Merge the authored change to `main`** (`clusters/base/cluster-vars.yaml`,
|
||||
`clusters/dev/`, `clusters/prod/`, the two Ansible files, `platform/externaldns`,
|
||||
`catalog/render.py`). With D1 in place nothing moves yet.
|
||||
- [ ] **D4. Release dev's reconciliation deliberately.** Resume the root Kustomization (or restore
|
||||
the in-cluster url, now pointing at `gitea.dev.home.keithjackson.dev`) and watch
|
||||
`flux get kustomizations -A` converge. Dev's Gateway listeners, HTTPRoutes, coredns-custom zone
|
||||
and Homepage links all move here. Dev's external-dns publishes the new record set under owner
|
||||
`dev`.
|
||||
- [ ] **D5. Keycloak: issuer + every client.** Dev's realm issuer becomes
|
||||
`https://keycloak.dev.home.keithjackson.dev/realms/homelab`. The issuer is baked into every
|
||||
token, and **~12 OIDC clients** need their redirect URIs moved in the same change: `grafana`,
|
||||
`gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, `qbittorrent`,
|
||||
`radarr`, `sonarr`, plus the `kubernetes` public client. Most are reconciled from
|
||||
`platform/keycloak/keycloak-realm-import.yaml` and move with D4; **headlamp's is the exception**
|
||||
— its issuer/callback live in a SOPS-encrypted Secret and must be re-encrypted by hand.
|
||||
- [ ] **D6. Rolling k3s restart + kubectl SSO re-issue.** `k3s_base_domain` now renders a different
|
||||
`oidc-issuer-url` into `/etc/rancher/k3s/config.yaml`, so the `k3s_server` role is **no longer
|
||||
a no-op on dev**: it rewrites the file and fires the Restart handler on all three servers. Run
|
||||
it **one server at a time**, confirming `kubectl get nodes` between each. Every user's OIDC
|
||||
kubeconfig is invalidated and must be re-issued; the cert-based admin kubeconfig keeps working
|
||||
throughout, which is what makes this step recoverable. **Do not reboot nodes** — restart the
|
||||
`k3s` service only.
|
||||
- [ ] **D7. Retire dev's old apex records.** They are owned by `default`, which no environment now
|
||||
claims, so nothing removes them automatically. Delete the `<svc>.home.keithjackson.dev` A
|
||||
records, the apex record, **and their TXT registry entries** (`external-dns-*` TXT), plus any
|
||||
stale `_acme-challenge.home.keithjackson.dev`. *Alternative, if you prefer external-dns to do
|
||||
its own cleanup:* set dev's `EXTERNALDNS_TXT_OWNER_ID` back to `default` for exactly one
|
||||
reconcile after D4 — `policy: sync` will then garbage-collect the apex set it no longer has
|
||||
sources for — and only then switch it to `dev`. That is two commits and needs watching; the
|
||||
direct deletion is preferred because it is verifiable in one place.
|
||||
- [ ] **D8. Verify the apex is EMPTY, against Cloudflare.** `dig +short <svc>.home.keithjackson.dev
|
||||
@1.1.1.1` returns nothing for every service name, and no TXT registry entry for those names
|
||||
remains. **This is the gate on the whole P sequence.** A pfSense/PiHole override is not
|
||||
evidence about public DNS.
|
||||
|
||||
### P — prod takes the apex
|
||||
|
||||
- [ ] **P1. Only now unsuspend prod**, in dependency order: `flux resume kustomization platform`,
|
||||
then the 10 downstream tiers. Confirm D8 first, in Cloudflare, not from a local resolver.
|
||||
- [ ] **P2. Prod's external-dns publishes under owner `prod`.** Check the logs for a matching hosted
|
||||
zone (F-DNS1 fail-closed mode: no zone selected ⇒ nothing created — which is safe, but means
|
||||
the filter was narrowed against instruction). Confirm the new TXT registry entries read owner
|
||||
`prod` and that no record owned by `dev` was touched.
|
||||
- [ ] **P3. Prod's wildcard cert `*.home.keithjackson.dev` + `home.keithjackson.dev` is `Ready`** and
|
||||
solved DNS-01 without colliding with any `_acme-challenge` dev still holds.
|
||||
- [ ] **P4. Prod's kubectl SSO works without an `/etc/hosts` pin.** Prod nodes resolve
|
||||
`keycloak.home.keithjackson.dev` through **public** upstreams (§4), so this only becomes true
|
||||
once P2 has published that name at prod's Gateway VIP (192.168.2.20). Before then a prod node
|
||||
resolving its own issuer reaches **dev**. Use the break-glass cert kubeconfig until P2 is done.
|
||||
- [ ] **P5. Cut prod's GitOps source over** from the workstation bootstrap Gitea
|
||||
(`http://192.168.2.132:3000/...`) to `https://gitea.home.keithjackson.dev/...` once prod runs
|
||||
its own Gitea — the block in `clusters/prod/kustomization.yaml` owns that procedure. Note the
|
||||
inversion **simplified** this: prod's apex Gitea name and dev's `gitea.dev.home...` are now
|
||||
different hosts, so prod pointing at the apex means "prod's own Gitea", never dev's.
|
||||
|
||||
### ACME: the duplicate-certificate limit
|
||||
|
||||
Let's Encrypt rate-limits **duplicate certificates — 5 per week for an identical set of names.**
|
||||
Prod's wildcard is `*.home.keithjackson.dev` + `home.keithjackson.dev`, which is **exactly the name
|
||||
set dev holds today**. Every prod order therefore counts against the same bucket as dev's historical
|
||||
renewals of that pair, and a retry loop during P3 can exhaust it and leave prod with **no trusted
|
||||
cert for up to a week**. Mitigations:
|
||||
|
||||
- Order prod's cert **after** D8, so dev is no longer renewing the same pair.
|
||||
- Let dev's old `*.home.keithjackson.dev` `Certificate`/Secret be **deleted** in D4/D7 rather than
|
||||
left to renew in the background.
|
||||
- Rehearse against **letsencrypt-staging** first (`ACME_CLUSTER_ISSUER`) — staging has its own,
|
||||
far looser limits — and only switch prod to `letsencrypt-prod` when the DNS-01 solve is proven.
|
||||
- Do **not** delete-and-recreate prod's `Certificate` to "retry"; each recreation is a fresh order.
|
||||
|
||||
### The L2 tests hardcode the apex
|
||||
|
||||
Six behavioural (L2) chainsaw tests embed `home.keithjackson.dev` as a literal:
|
||||
|
||||
| test | file |
|
||||
|---|---|
|
||||
| `dns-exposure` | `conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml` |
|
||||
| `dns-internal-resolution` | `conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml` |
|
||||
| `gateway-ingress-reachable` | `conformance/tests/l2-behavioral/gateway-ingress-reachable/chainsaw-test.yaml` (+ `fixtures/app-and-probe.yaml`) |
|
||||
| `oidc-sso-grafana` | `conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml` |
|
||||
| `sso-forward-auth` | `conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml` |
|
||||
| `trusted-tls` | `conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml` |
|
||||
|
||||
They are **test inputs, not reconciled manifests**, so they are deliberately left alone by the
|
||||
authoring change: they are the only thing still asserting the apex, and while dev is live on the apex
|
||||
they are *correct*. Under the new direction the literal happens to become right for **prod** and
|
||||
wrong for **dev**, so they must be **parameterized per environment** (not merely edited) before
|
||||
either §9's *"suite passes green against the prod cluster"* or a green L2 run against migrated dev
|
||||
can be claimed. Until then, expect the dev L2 suite to fail between D4 and that parameterization —
|
||||
a known, scheduled failure, not a regression.
|
||||
|
||||
The L1 *rendering* tests need no change: their goldens carry `${BASE_DOMAIN}` as a token, and the
|
||||
literal hostnames in them come from each test's own fixture `ingressRoutes[].host`, which is
|
||||
environment-independent by construction.
|
||||
|
||||
### Offline proof carried out for the authoring change (no cluster touched)
|
||||
|
||||
All 12 `substituteFrom` paths were rendered for **both** overlays through the real toolchain
|
||||
(`kubectl kustomize --load-restrictor LoadRestrictionsNone <spec.path> | flux envsubst --strict`,
|
||||
with each overlay's effective `cluster-vars` taken from `kubectl kustomize clusters/<env>`), before
|
||||
and after the change.
|
||||
|
||||
- **Dev's diff is *only* the hostname move.** Proven by inverse substitution rather than by reading
|
||||
the diff: applying `s/dev.home.keithjackson.dev/home.keithjackson.dev/` to the new render restores
|
||||
the baseline **byte-for-byte** in all 12 paths, plus 2 declared `txtOwnerId: default → dev` lines.
|
||||
145 changed line pairs, all classified: 143 hostname-move, 2 declared owner-id, **0 unexplained**,
|
||||
and **0 bare-apex remnants**. The 2 paths with no change (`cilium`, `velero`) provably contain no
|
||||
hostname at all (`grep -c keithjackson.dev` = 0).
|
||||
- **Prod lands on the bare apex:** **0** occurrences of `prod.home.keithjackson.dev` and **0** of
|
||||
`dev.home.keithjackson.dev` anywhere in prod's rendered output; 143 apex names.
|
||||
- **Certs:** dev `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev`; prod
|
||||
`*.home.keithjackson.dev` + `home.keithjackson.dev`.
|
||||
- **Positive controls** (the harness is provably not blind): perturbing an unrelated knob
|
||||
(`GATEWAY_VIP`) surfaces 1 unexplained line; perturbing `BASE_DOMAIN` to a wrong value fails 10 of
|
||||
12 paths; injecting a single un-moved hostname is caught by the independent bare-apex scan (which
|
||||
the byte-identity check alone cannot see — which is why both exist).
|
||||
- Gates: `ci/l0.sh`, `ci/l1.sh`, `ci/render-golden.sh`, `ci/lint.sh`,
|
||||
`flux-postbuild-substitution/check.sh` (254 assertions), `catalog/render.py --selftest`,
|
||||
`ci/spec-immutability.sh` — all **PASS**.
|
||||
@@ -289,15 +289,70 @@ node_baseline_dns_servers:
|
||||
# LAN clients -> PiHole. Not a cluster concern.
|
||||
# nodes -> THIS FILE. Public resolvers, so the zone resolves PUBLICLY.
|
||||
#
|
||||
# THE ONE THING THAT BREAKS: the k3s API SERVER is a node process and carries
|
||||
# `oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab`
|
||||
# (roles/k3s_server/tasks/main.yml). It reaches the issuer through the NODE resolver, so on
|
||||
# prod it would get the PUBLIC record instead of the prod Gateway VIP — breaking kubectl
|
||||
# OIDC SSO. Dev avoids this because its node resolver is PiHole, which answers the zone
|
||||
# with the dev Gateway VIP.
|
||||
# THE k3s API SERVER is a node process and carries `oidc-issuer-url=...` (built from
|
||||
# k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through
|
||||
# these PUBLIC upstreams — not through CoreDNS's split-horizon answer.
|
||||
#
|
||||
# NOT blocking today (prod has no cluster and no Keycloak yet). Options, in
|
||||
# PROD-PROVISIONING.md §4: a narrow Ansible-managed /etc/hosts entry pinning
|
||||
# keycloak.home.keithjackson.dev -> 192.168.2.20 on prod nodes; or a local resolver on the
|
||||
# prod LAN serving the zone; or run prod on the cert kubeconfig and defer OIDC.
|
||||
# ORDERING HAZARD, NOT YET CLEARED. Prod's issuer is keycloak.home.keithjackson.dev — an APEX
|
||||
# name, because prod IS the apex under the 2026-09-04 inversion (k3s_base_domain below).
|
||||
# That is the name the LIVE DEV CLUSTER answers today, and its public A record currently
|
||||
# points at DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. A prod
|
||||
# node resolving its own issuer through these public upstreams therefore reaches dev until:
|
||||
# (1) dev has migrated to *.dev.home.keithjackson.dev and RETIRED its apex records, and
|
||||
# (2) PROD's own external-dns has published keycloak.home.keithjackson.dev at PROD's
|
||||
# Gateway VIP (192.168.2.20).
|
||||
# Only then do the public answer and the in-cluster split-horizon answer AGREE, and no
|
||||
# /etc/hosts pin is needed. Until then kubectl OIDC SSO on prod cannot work; the cert-based
|
||||
# admin kubeconfig is the break-glass path and always works. Ordering, not a design gap —
|
||||
# the ordered steps are infra/PROD-PROVISIONING.md §10.
|
||||
node_baseline_dns_search: []
|
||||
|
||||
# --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) --------------------
|
||||
# PROD **IS** THE APEX. [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml —
|
||||
# the same one-fact-in-two-places contract as k3s_data_dir and k3s_api_host. The Flux side
|
||||
# cannot supply this: the kube-apiserver's oidc-issuer-url is a k3s server FLAG written to
|
||||
# /etc/rancher/k3s/config.yaml, not a manifest, so postBuild.substituteFrom never sees it.
|
||||
# Full rationale in roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url derives.
|
||||
#
|
||||
# WHY THE SUFFIXES SPLIT (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve
|
||||
# the SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
|
||||
# deleted every one of dev's public records.
|
||||
#
|
||||
# WHICH WAY ROUND — INVERTED 2026-09-04 by operator decision, superseding the interim split
|
||||
# committed earlier the same day (prod on `prod.home.` behind apex CNAMEs). The simpler target:
|
||||
# prod -> home.keithjackson.dev (THE APEX; this variable — no `prod.` label, no CNAMEs)
|
||||
# dev -> dev.home.keithjackson.dev (roles/k3s_server/defaults/main.yml)
|
||||
# So prod's Keycloak issuer is https://keycloak.home.keithjackson.dev/realms/homelab, and the
|
||||
# `kubernetes` OIDC client in PROD's OWN Keycloak realm is what prod's kubectl SSO authenticates
|
||||
# against.
|
||||
#
|
||||
# ORDERING HAZARD SPECIFIC TO THIS VARIABLE. keycloak.home.keithjackson.dev is the name the LIVE
|
||||
# dev cluster answers today, and prod's nodes resolve the issuer through PUBLIC upstreams (see
|
||||
# node_baseline_dns_search above). So until dev has vacated the apex and PROD's external-dns has
|
||||
# published that name at PROD's Gateway VIP (192.168.2.20), a prod node resolving its own issuer
|
||||
# gets DEV's VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. kubectl OIDC SSO on prod
|
||||
# therefore does not work until §10's dev-vacate steps are complete; the cert-based admin
|
||||
# kubeconfig is the break-glass path and always works. Ordering, not a design gap.
|
||||
k3s_base_domain: "home.keithjackson.dev"
|
||||
|
||||
# --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) -----
|
||||
# roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE
|
||||
# and group_vars/all.yml carries no override, so dev keeps resolving exactly that).
|
||||
# Prod bootstraps its OWN overlay. Before this seam existed the role could bootstrap
|
||||
# clusters/dev ONLY — bootstrap/flux/bootstrap.sh's FLUX_ENV_PATH default was never
|
||||
# exported by the role — so `flux bootstrap ... --path=clusters/prod` (§8 step 4 of
|
||||
# infra/PROD-PROVISIONING.md) was simply not reachable without editing the role.
|
||||
#
|
||||
# Same seam as kube_vip_address / k3s_api_host / k3s_data_dir / cluster_kubeconfig_name /
|
||||
# node_baseline_backports_kernel above: set once for the [k3s_prod] umbrella group, so a
|
||||
# FOURTH prod node inherits it from group membership with no per-host or per-run flag.
|
||||
#
|
||||
# WHAT clusters/prod STILL NEEDS before this is safe to run — do NOT invoke the
|
||||
# flux_bootstrap role until these are true (PROD-PROVISIONING.md §7):
|
||||
# - the remaining TBD-PROD-* placeholders in clusters/prod/kustomization.yaml are real
|
||||
# (S3_BACKUP_ENDPOINT); they are INVALID on purpose so an early apply fails loudly;
|
||||
# - the GitRepository url there is the TEMPORARY workstation Gitea
|
||||
# (http://192.168.2.132:3000/...) and that container is actually running and reachable
|
||||
# from the prod nodes — it is outside every cluster and can simply be powered off;
|
||||
# - the `sops-age` Secret is installed out of band on prod (bootstrap/sops).
|
||||
flux_bootstrap_env_path: "clusters/prod"
|
||||
@@ -15,9 +15,20 @@ cilium_namespace: "kube-system"
|
||||
# Cilium 1.20.1 targets Gateway API v1.6.1.
|
||||
gateway_api_version: "v1.6.1"
|
||||
gateway_api_crd_urls:
|
||||
# ONE URL, deliberately. As of v1.6.0 the STANDARD channel bundle already contains
|
||||
# TLSRoute (plus TCPRoute, UDPRoute, ListenerSet and BackendTLSPolicy), so the separate
|
||||
# experimental TLSRoute manifest this list used to carry is redundant — and worse than
|
||||
# redundant: v1.6 also ships a ValidatingAdmissionPolicy,
|
||||
# `safe-upgrades.gateway.networking.k8s.io`, which REJECTS an experimental-channel CRD
|
||||
# applied over a standard-channel one. Applying the old second URL therefore fails hard:
|
||||
# ValidatingAdmissionPolicy 'safe-upgrades.gateway.networking.k8s.io' ... denied request:
|
||||
# Installing experimental CRDs on top of standard channel CRDs is prohibited by default.
|
||||
# Verified 2026-09-04 on the prod bootstrap: standard-install.yaml v1.6.1 alone produces the
|
||||
# exact 10 gateway.networking.k8s.io CRDs the LIVE dev cluster is running, tlsroutes included
|
||||
# and annotated channel=standard, bundle-version=v1.6.1 — so this is parity with dev, not a
|
||||
# reduction. Dev needs no reapply; it already has them. If gateway_api_version is ever moved
|
||||
# BACK below v1.6.0, the experimental TLSRoute URL has to come back with it.
|
||||
- "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{ gateway_api_version }}/standard-install.yaml"
|
||||
# TLSRoute (experimental) is required by Cilium's Gateway API support.
|
||||
- "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{ gateway_api_version }}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"
|
||||
|
||||
# Local kubeconfig fetched from the primary. *.kubeconfig is gitignored. The FILENAME
|
||||
# comes from cluster_kubeconfig_name (group_vars) so a prod run cannot overwrite the dev
|
||||
|
||||
@@ -19,5 +19,22 @@ flux_bootstrap_repo_root: "{{ playbook_dir }}/../../.."
|
||||
flux_bootstrap_sops_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/sops/install-age-key.sh"
|
||||
|
||||
# Assumed interface of bootstrap/flux (GitOps Engineer, T-1.4):
|
||||
# runs `flux bootstrap ... --path=clusters/dev` against this cluster.
|
||||
# runs `flux install` + `kubectl apply -k <env path>` against this cluster.
|
||||
flux_bootstrap_flux_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/flux/bootstrap.sh"
|
||||
|
||||
# --- WHICH ENVIRONMENT OVERLAY THIS BOOTSTRAPS (R-ENV-02) -------------------------
|
||||
# The one thing that makes this role environment-agnostic. bootstrap/flux/bootstrap.sh
|
||||
# reads FLUX_ENV_PATH (default clusters/dev) and ends with `kubectl apply -k <that>`;
|
||||
# tasks/main.yml exports this value into that variable, so the role NEVER names an
|
||||
# environment and the task graph is identical for dev and prod.
|
||||
#
|
||||
# THE DEFAULT IS DEV ON PURPOSE — dev is LIVE, and group_vars/all.yml carries no
|
||||
# override, so a dev run resolves exactly the value it always had (clusters/dev) and
|
||||
# behaves byte-for-byte as before.
|
||||
#
|
||||
# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same group_vars
|
||||
# seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name /
|
||||
# node_baseline_backports_kernel. No role fork, no per-run extra-var, no second
|
||||
# playbook: group membership decides. Before this existed the role could bootstrap
|
||||
# ONLY clusters/dev, so prod bring-up was impossible without editing the role.
|
||||
flux_bootstrap_env_path: "clusters/dev"
|
||||
@@ -4,11 +4,17 @@
|
||||
# — it calls the entrypoints the Security + GitOps Engineers author there.
|
||||
#
|
||||
# Order matters: install the age decryption key FIRST (so Flux can decrypt on its
|
||||
# very first reconcile), THEN run `flux bootstrap` at clusters/dev.
|
||||
# very first reconcile), THEN bootstrap Flux at the environment's overlay.
|
||||
#
|
||||
# The two commands are variables (see defaults/main.yml). If the bootstrap workers
|
||||
# expose different entrypoints, override flux_bootstrap_sops_cmd / _flux_cmd — the
|
||||
# task graph does not change.
|
||||
#
|
||||
# ENVIRONMENT-AGNOSTIC (R-ENV-02): this role names NO environment. Which overlay gets
|
||||
# bootstrapped comes from `flux_bootstrap_env_path` (defaults/main.yml, default
|
||||
# clusters/dev), exported below as FLUX_ENV_PATH — the variable bootstrap/flux/
|
||||
# bootstrap.sh already reads. Prod overrides it in group_vars/k3s_prod/vars.yml, the
|
||||
# same seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name.
|
||||
|
||||
- name: Assert the bootstrap/ subtree exists (authored by the other worker)
|
||||
ansible.builtin.stat:
|
||||
@@ -29,11 +35,14 @@
|
||||
run_once: true
|
||||
changed_when: true
|
||||
|
||||
- name: Bootstrap Flux against clusters/dev (bootstrap/flux)
|
||||
- name: Bootstrap Flux against this environment's overlay (bootstrap/flux)
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ flux_bootstrap_flux_cmd }}"
|
||||
environment:
|
||||
KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}"
|
||||
# The ONLY per-environment input. clusters/dev by default (dev unchanged);
|
||||
# clusters/prod for [k3s_prod] hosts via group_vars.
|
||||
FLUX_ENV_PATH: "{{ flux_bootstrap_env_path }}"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
|
||||
@@ -66,3 +66,44 @@ k3s_tls_sans: >-
|
||||
{{ ([k3s_api_host, kube_vip_address, '127.0.0.1']
|
||||
+ (groups[k3s_servers_group] | map('extract', hostvars, 'k3s_node_ip') | list))
|
||||
| unique }}
|
||||
|
||||
# --- Keycloak OIDC issuer for kubectl SSO (R-IDP-03, R-ENV-02) -----------------------
|
||||
# THE ANSIBLE-SIDE HALF OF ${BASE_DOMAIN}, and it cannot be the Flux-side half.
|
||||
#
|
||||
# The kube-apiserver's `oidc-issuer-url` is a k3s SERVER FLAG written into
|
||||
# /etc/rancher/k3s/config.yaml by this role — it is NOT a Kubernetes manifest, so Flux never
|
||||
# sees it and `postBuild.substituteFrom: cluster-vars` cannot reach it. The platform hostname
|
||||
# suffix therefore exists as ONE FACT IN TWO PLACES, and they must agree:
|
||||
#
|
||||
# Flux side clusters/base/cluster-vars.yaml BASE_DOMAIN (per-overlay)
|
||||
# Ansible side THIS variable k3s_base_domain (per-inventory-group)
|
||||
#
|
||||
# [parity] k3s_base_domain MUST equal the BASE_DOMAIN the SAME cluster's Flux overlay sets.
|
||||
# If they disagree, the apiserver validates ID tokens against a DIFFERENT Keycloak than the
|
||||
# one the cluster actually runs: every `kubectl` SSO login fails with an issuer mismatch,
|
||||
# while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken",
|
||||
# not like a config skew. The break-glass cert kubeconfig is why this is recoverable.
|
||||
#
|
||||
# THE DEFAULT IS THE DEV VALUE — the same convention as k3s_data_dir above and cluster-vars'
|
||||
# GATEWAY_VIP / LONGHORN_DATA_PATH: the role default is dev's value and prod overrides it in
|
||||
# inventory/group_vars/k3s_prod/vars.yml (the same inventory seam k3s_data_dir / k3s_api_host /
|
||||
# kube_vip_address use, so a FOURTH prod server inherits it from group membership rather than
|
||||
# from anyone remembering).
|
||||
#
|
||||
# *** WHICH DEV VALUE — UPDATED 2026-09-04, AND IT IS AHEAD OF THE LIVE DEV CLUSTER. ***
|
||||
# The operator inverted the suffix split: PROD is now the apex `home.keithjackson.dev` and DEV
|
||||
# carries the label `dev.home.keithjackson.dev`. This default therefore moved from the apex to
|
||||
# dev's subdomain, mirroring BASE_DOMAIN in clusters/dev/kustomization.yaml.
|
||||
#
|
||||
# CONSEQUENCE — this default no longer matches what the live dev nodes are running, so unlike
|
||||
# every previous edit to this file A DEV RE-RUN IS NOT A NO-OP. Rendering config.yaml with the
|
||||
# new issuer CHANGES the file, FIRES the Restart handler, and rolls k3s on all three dev
|
||||
# servers, which also invalidates every user's OIDC kubeconfig (SSO must be re-issued). That is
|
||||
# a deliberate step of the dev DNS migration, NOT a side effect to discover during unrelated
|
||||
# work: DO NOT run the k3s_server role against dev until step D6 of
|
||||
# infra/PROD-PROVISIONING.md §10, and never while the operator is away. The break-glass
|
||||
# cert-based admin kubeconfig keeps working throughout, which is what makes the step safe.
|
||||
k3s_base_domain: "dev.home.keithjackson.dev"
|
||||
|
||||
# Derived, so the realm path is stated once. Override k3s_base_domain, not this.
|
||||
k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab"
|
||||
@@ -75,13 +75,19 @@
|
||||
# OIDC (Keycloak SSO for kubectl). The API server trusts ID tokens from the homelab realm;
|
||||
# the `kubernetes` PUBLIC client (PKCE) + kubelogin mint them, and RBAC binds the `groups`
|
||||
# claim (platform/keycloak/k8s-oidc-rbac.yaml: oidc:platform-admins->cluster-admin,
|
||||
# oidc:viewers->view). Nodes resolve keycloak.home.keithjackson.dev to the Cilium gateway VIP
|
||||
# (192.168.1.53) and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:`
|
||||
# oidc:viewers->view). Nodes resolve keycloak.<k3s_base_domain> to the Cilium gateway VIP
|
||||
# and trust its public LE-prod cert, so NO oidc-ca-file is needed. The `oidc:`
|
||||
# prefixes namespace federated identities away from cert / service-account subjects.
|
||||
# ADDITIVE + break-glass-safe: the cert-based admin kubeconfig keeps full access with or
|
||||
# without Keycloak, so a bad realm/OIDC state can never lock you out of the cluster.
|
||||
#
|
||||
# NOT a Flux ${BASE_DOMAIN} token: this is a k3s SERVER FLAG, not a manifest, so
|
||||
# postBuild.substituteFrom never touches it. k3s_oidc_issuer_url derives from
|
||||
# k3s_base_domain (defaults/main.yml), whose [parity] contract with the cluster's
|
||||
# BASE_DOMAIN is documented there. Dev's default renders the value it already runs, so
|
||||
# config.yaml is unchanged and the Restart handler does not fire.
|
||||
kube-apiserver-arg:
|
||||
- "oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab"
|
||||
- "oidc-issuer-url={{ k3s_oidc_issuer_url }}"
|
||||
- "oidc-client-id=kubernetes"
|
||||
- "oidc-username-claim=email"
|
||||
- "oidc-username-prefix=oidc:"
|
||||
@@ -98,15 +104,20 @@
|
||||
INSTALL_K3S_VERSION: "{{ k3s_version }}"
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
# PATH FOLLOWS --data-dir. k3s writes the node-token to <data-dir>/server/token, NOT to
|
||||
# the literal /var/lib/rancher/k3s. On dev k3s_data_dir IS /var/lib/rancher/k3s (the role
|
||||
# default, restated explicitly above) so this is byte-for-byte the old path; on prod
|
||||
# --data-dir is /data/k3s (group_vars/k3s_prod) and the hardcoded path did not exist at all
|
||||
# — this task would have timed out after 180s and the two secondaries could never join.
|
||||
- name: Wait for the k3s node-token to exist (primary)
|
||||
ansible.builtin.wait_for:
|
||||
path: /var/lib/rancher/k3s/server/token
|
||||
path: "{{ k3s_data_dir }}/server/token"
|
||||
timeout: 180
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
- name: Read the k3s node-token (primary)
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/token
|
||||
src: "{{ k3s_data_dir }}/server/token"
|
||||
register: k3s_token_raw
|
||||
when: k3s_server_mode == 'primary'
|
||||
|
||||
@@ -126,9 +137,11 @@
|
||||
# "The primary" is resolved through k3s_primary_group (group_vars: `k3s_primary` for
|
||||
# dev, `k3s_prod_primary` for prod), never a literal group name — so a prod secondary
|
||||
# delegates to the PROD primary using this same role, unchanged (R-ENV-02).
|
||||
# k3s_data_dir is a CLUSTER-WIDE value (group_vars for prod, role default for dev), so the
|
||||
# secondary's own copy of it is also the primary's data dir — correct for the delegated read.
|
||||
- name: Fetch the k3s node-token directly from the primary (secondary)
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/rancher/k3s/server/token
|
||||
src: "{{ k3s_data_dir }}/server/token"
|
||||
delegate_to: "{{ groups[k3s_primary_group][0] }}"
|
||||
register: k3s_primary_token_raw
|
||||
when:
|
||||
|
||||
@@ -31,4 +31,14 @@ kube_vip_retry_period: 2
|
||||
|
||||
# k3s auto-deploy manifests dir: k3s applies anything here on start, so kube-vip
|
||||
# is reproducible and independent of Flux/Helm.
|
||||
kube_vip_manifest_path: "/var/lib/rancher/k3s/server/manifests/kube-vip.yaml"
|
||||
#
|
||||
# THE PATH FOLLOWS --data-dir. k3s reads auto-deploy manifests from
|
||||
# <data-dir>/server/manifests, so this must be derived from the cluster-wide
|
||||
# k3s_data_dir (roles/k3s_server turns the same variable into --data-dir), not from the
|
||||
# literal /var/lib/rancher/k3s. Dev leaves k3s_data_dir unset outside roles/k3s_server's
|
||||
# own defaults, so the fallback below reproduces dev's existing path exactly; prod sets
|
||||
# k3s_data_dir: /data/k3s in group_vars/k3s_prod, where the old hardcoded path was simply
|
||||
# not the directory k3s watches — the manifest would have been written and silently never
|
||||
# applied, and the control-plane VIP would never have come up.
|
||||
kube_vip_manifest_path: >-
|
||||
{{ k3s_data_dir | default('/var/lib/rancher/k3s') }}/server/manifests/kube-vip.yaml
|
||||
@@ -1,13 +1,22 @@
|
||||
---
|
||||
# Deploy kube-vip (control-plane HA VIP) via the k3s auto-deploy manifests dir.
|
||||
# k3s applies /var/lib/rancher/k3s/server/manifests/*.yaml on start, so this is
|
||||
# reproducible and independent of Flux. The same manifest is written on every
|
||||
# k3s applies <data-dir>/server/manifests/*.yaml on start, so this is
|
||||
# reproducible and independent of Flux (see kube_vip_manifest_path in defaults/main.yml —
|
||||
# the dir follows k3s's --data-dir, which prod moves to /data/k3s). The same manifest is written on every
|
||||
# server so the DaemonSet survives the loss of any single node.
|
||||
|
||||
- name: Detect the LAN interface holding the node's 192.168.1.5x address
|
||||
# WHICH INTERFACE kube-vip ARPs THE VIP ON. Detected from this node's OWN k3s_node_ip
|
||||
# (inventory, per host) rather than a hardcoded subnet: the previous awk matched the
|
||||
# literal 192.168.1.5[0-2], i.e. DEV's node addresses, so on prod (192.168.2.10/.11/.12)
|
||||
# it returned nothing and the assert below aborted the play. k3s_node_ip is the same
|
||||
# address k3s advertises with --node-ip and is defined for dev and prod alike, so this
|
||||
# resolves eno1 on prod and the identical dev interface it always resolved before.
|
||||
# `ip addr show to <ip>/32` does the matching in iproute2 instead of a regex over dotted
|
||||
# quads (where `.` would match any character).
|
||||
- name: Detect the LAN interface holding this node's k3s_node_ip
|
||||
ansible.builtin.shell: >-
|
||||
set -o pipefail;
|
||||
ip -o -4 addr show | awk '/192\.168\.1\.5[0-2]\//{print $2; exit}'
|
||||
ip -o -4 addr show to {{ k3s_node_ip }}/32 | awk '{print $2; exit}'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: kube_vip_iface_detect
|
||||
@@ -25,7 +34,7 @@
|
||||
- kube_vip_vip_interface is defined
|
||||
- kube_vip_vip_interface | length > 0
|
||||
fail_msg: >-
|
||||
Could not detect the 192.168.1.5x LAN interface for kube-vip on
|
||||
Could not detect the interface holding {{ k3s_node_ip }} for kube-vip on
|
||||
{{ inventory_hostname }} — set kube_vip_interface explicitly.
|
||||
|
||||
- name: Ensure the k3s auto-deploy manifests directory exists
|
||||
|
||||
@@ -183,6 +183,29 @@ node_baseline_protected_mounts:
|
||||
- /home
|
||||
- /tmp
|
||||
|
||||
# --- WIRELESS (tasks/main.yml, tag `wireless`) ---------------------------------------
|
||||
# A CLUSTER NODE GETS EXACTLY ONE PATH ONTO THE NETWORK. ON by default — unlike the
|
||||
# backports kernel above, which is a prod-only deviation, this is a cluster-wide policy and
|
||||
# it is a provable NO-OP where it does not apply: the tasks act only on interfaces the node
|
||||
# reports as real 802.11 devices (a `phy80211` link in sysfs), and dev's Proxmox VMs have
|
||||
# none, so on dev nothing is written, nothing is unloaded and nothing changes. Defaulting it
|
||||
# true is what makes a FOURTH node with a wifi card inherit the policy from being a cluster
|
||||
# node, rather than from someone remembering to set a flag.
|
||||
#
|
||||
# WHY IT EXISTS: k3s-prod-3 came up on both eno1 (192.168.2.12, the inventory address) and
|
||||
# wlp1s0 (192.168.2.195, DHCP on the same /24). The wifi lease's dhcpcd run rewrote
|
||||
# /etc/resolv.conf three minutes after this role wrote it, and a second address on the LAN
|
||||
# where kube-vip floats 192.168.2.9 and Cilium L2-announces .20-.30 is a problem in its own
|
||||
# right. See the `wireless` block in tasks/main.yml for the full reasoning, including why the
|
||||
# DRIVER MODULE is the layer that gets blocked rather than the link, a service, or rfkill.
|
||||
#
|
||||
# SET IT FALSE for a host that must genuinely keep wifi (there is none today). That does NOT
|
||||
# undo an already-applied block — delete the file below on the node as well.
|
||||
node_baseline_disable_wireless: true
|
||||
# Where the persistent block lives. 99- so it is read last and overrides any earlier
|
||||
# permissive rule; the name says what it does, on a node with no repo checkout to hand.
|
||||
node_baseline_wireless_modprobe_file: /etc/modprobe.d/99-k3s-no-wireless.conf
|
||||
|
||||
# --- DNS resilience (R-DNS-01 / R-DNS-03) -------------------------------------------
|
||||
# These are the DEV values. Prod overrides them in inventory/group_vars/k3s_prod/vars.yml
|
||||
# (the same seam kube_vip_address / k3s_api_host / cluster_kubeconfig_name use) — NOT in
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
# TAGS — every task here carries one of `packages`, `kernel`, `system`, `disks` or `dns`. An
|
||||
# untagged run is unchanged (all five run, in this order). The split exists because the PACKAGE floor
|
||||
# TAGS — every task here carries one of `packages`, `kernel`, `system`, `wireless`, `disks` or
|
||||
# `dns`. An untagged run is unchanged (all six run, in this order). The split exists because the PACKAGE floor
|
||||
# and the DATA-DISK work have different readiness: packages are safe on any node from
|
||||
# first contact, while the disk tasks need block devices that a new environment may not
|
||||
# have decided yet (prod: infra/PROD-PROVISIONING.md §2, whose decision is itself blocked
|
||||
@@ -16,6 +16,12 @@
|
||||
# is: it is the ONE thing that must land on a prod node BEFORE k3s is installed (§0b), and
|
||||
# it must be runnable without touching a disk decision that is still open:
|
||||
# ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags kernel
|
||||
# `wireless` (a cluster node gets exactly ONE path onto the network) is separated for a third
|
||||
# reason: it must be runnable BEFORE `dns` and without touching packages, the kernel or a disk
|
||||
# decision, because a node with a second DHCP-managed interface reverts the `dns` write. It is
|
||||
# placed before the disk work for the same reason, and the pair is the whole remedy for the
|
||||
# k3s-prod-3 incident:
|
||||
# ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags wireless,dns
|
||||
|
||||
- name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs)
|
||||
ansible.builtin.apt:
|
||||
@@ -314,6 +320,304 @@
|
||||
state: "{{ 'started' if node_baseline_qga_channel.stat.exists else 'stopped' }}"
|
||||
tags: [system]
|
||||
|
||||
# =====================================================================================
|
||||
# WIRELESS — tag `wireless`: a cluster node gets exactly ONE path onto the network
|
||||
# =====================================================================================
|
||||
# WHAT WENT WRONG (k3s-prod-3, reported by the operator 2026-09-04). The node was up on BOTH
|
||||
# eno1 (192.168.2.12 — the inventory address) and wlp1s0 (192.168.2.195, DHCP, the SAME /24).
|
||||
# The wifi lease carried a dhcpcd run with it, and dhcpcd's `20-resolv.conf` hook REWROTE
|
||||
# /etc/resolv.conf, undoing the resolvers this role had just written: the file read
|
||||
# "# Generated by dhcpcd from wlp1s0.dhcp, wlp1s0.dhcp6, wlp1s0.ra" with nameserver
|
||||
# 192.168.2.1 plus two Google IPv6 servers, timestamped THREE MINUTES AFTER the role's own
|
||||
# write (01:32 -> 01:35). The role had reported `changed` and green. That is exactly the
|
||||
# "successful run that fixed nothing" the `contended` branch of the DNS block below exists to
|
||||
# prevent, and the detection blind spot that let it through is closed there.
|
||||
#
|
||||
# WHY THE INTERFACE IS REMOVED RATHER THAN JUST REPORTED. Closing the detection hole alone
|
||||
# would have turned a silent revert into a loud failure — an improvement, but it leaves the
|
||||
# node with a second, DHCP-managed path onto the LAN, and on a k3s server that is three
|
||||
# distinct problems beyond DNS:
|
||||
# * a SECOND DEFAULT ROUTE, whose metric decides which source address the node uses;
|
||||
# * a SECOND ADDRESS ON THE SAME L2 — the segment where kube-vip floats 192.168.2.9 and
|
||||
# Cilium L2-announces 192.168.2.20-.30 (§4). One host answering ARP on one subnet from
|
||||
# two NICs is the exact shape neither mechanism can reason about;
|
||||
# * an UNMANAGED address: nothing in this repo knows about .195, and the DHCP server is
|
||||
# free to hand out a different one after any lease expiry.
|
||||
# None of that is a resolver bug, and none of it is fixed by writing resolv.conf harder.
|
||||
#
|
||||
# WHY THE DRIVER LAYER — the four alternatives, and why each is weaker. What has to hold is:
|
||||
# survives a REBOOT, survives a LEASE RENEWAL, and cannot be undone by a routine `ifup`.
|
||||
# Measured on the nodes themselves — all three are HP EliteDesk 800 G5 minis, eno1 is
|
||||
# `e1000e`, the wifi NIC is an Intel AX200 on `iwlwifi`, and the OS is Debian 13 trixie with
|
||||
# ifupdown + `networking.service`, NO NetworkManager, NO systemd-networkd, NO systemd-resolved:
|
||||
# `ip link set <wifi> down` — NOT persistent. Any `ifup`, any hotplug event, any reboot
|
||||
# undoes it. It is what had already been done by hand on node 3, and the stale dhcpcd
|
||||
# /etc/resolv.conf outlived it — the link was down and the wrong resolvers were still
|
||||
# in the file, across a reboot.
|
||||
# mask the service that manages it — THERE IS NO SUCH SERVICE HERE. dhcpcd on these nodes
|
||||
# is `dhcpcd-base`: no dhcpcd.service exists at all (`systemctl is-enabled dhcpcd` =
|
||||
# not-found), it is spawned per-interface by ifupdown. The only unit that can bring
|
||||
# wireless up is `networking.service` — which ALSO owns eno1's static address, so
|
||||
# masking it would cut the one SSH path to a machine whose operator is not on site.
|
||||
# `rfkill` soft-block — weaker on every axis. The netdev still EXISTS, so ifupdown and
|
||||
# dhcpcd can still be pointed at it; persistence depends on systemd-rfkill restoring
|
||||
# saved state (the kernel's own `persistent` flag reads 0 on these devices, and the
|
||||
# `rfkill` CLI is not even installed); and one `rfkill unblock wlan` undoes it.
|
||||
# the BIOS `M.2 WLAN/BT` toggle — genuinely the deepest layer, and it reads `Enable` on all
|
||||
# three nodes (hp-bioscfg survey, §1b). NOT taken here: it needs a REBOOT to take
|
||||
# effect, and a prod reboot is a serialized, supervised operator step (§0b "SECOND
|
||||
# FAULT" — k3s-prod-2 hung on a warm reboot and needed the power button). It also
|
||||
# disables Bluetooth on the same M.2 card. Left as an operator recommendation in §4.
|
||||
# THE DRIVER MODULE — chosen. `blacklist` plus `install <module> /bin/false` in
|
||||
# /etc/modprobe.d means kmod refuses to load the wifi driver on this boot and every
|
||||
# future one, so THE NETDEV NEVER EXISTS. No interface means no DHCP lease, no
|
||||
# resolv.conf hook, no second route, no second address — and every re-activation path
|
||||
# above (udev hotplug, a hand-run `ifup`, wpa_supplicant, a hand-run dhcpcd) needs a
|
||||
# device that is not there. `blacklist` alone would still allow a dependency- or
|
||||
# hand-triggered `modprobe`; the `install ... /bin/false` line is what makes it refuse
|
||||
# outright. It is ONE declarative file, readable on the node, and undone by deleting
|
||||
# it. Paired with the runtime unload below, it needs NO reboot to take effect.
|
||||
#
|
||||
# SAFE AND IDEMPOTENT ON ALL THREE PROD NODES *AND* ON THE LIVE DEV CLUSTER. Nothing here is
|
||||
# hostname- or hardware-specific: the block acts only on interfaces the node itself reports as
|
||||
# real 802.11 devices (a `phy80211` link in sysfs) and on the driver modules behind them.
|
||||
# * dev (Proxmox VMs, virtio NICs) has no such interface, so the discovery comes back empty,
|
||||
# every task below is skipped, and NOT EVEN THE modprobe.d FILE IS WRITTEN. The block
|
||||
# cannot touch dev's networking, which is why it is safe to have on by default.
|
||||
# * k3s-prod-1 (wlp1s0) and k3s-prod-2 (wlp2s0) already have the link DOWN, so the link-down
|
||||
# task is skipped by its own `when` and only the modprobe.d file and the module unload
|
||||
# remain — and both are no-ops on a re-run.
|
||||
# * k3s-prod-3 (wlp1s0) is the reported node.
|
||||
# A re-run changes nothing on any of them: the file is content-identical, `ip link set down`
|
||||
# is gated on the link not already being down, and the unload is gated on the module still
|
||||
# being loaded.
|
||||
#
|
||||
# AND IT REFUSES TO CUT ITS OWN PATH IN. The assert below stops the play if the address
|
||||
# Ansible is connected to is bound to a wireless interface, or if the default route leaves
|
||||
# over one, or if the wireless driver module is also the driver of the default-route
|
||||
# interface. That is the one case where doing this would lock everybody out of a machine
|
||||
# nobody can walk up to, so it is checked per node, on every run, from that node's own facts.
|
||||
- name: Survey this node's 802.11 devices, their drivers and any live DHCP client
|
||||
ansible.builtin.shell:
|
||||
executable: /bin/sh
|
||||
cmd: |
|
||||
set -u
|
||||
# Flat KEY|value lines so the Ansible side needs one single-group regex per fact and
|
||||
# no parsing cleverness. A wireless interface is one with a phy80211 link in sysfs —
|
||||
# that is the kernel's own answer to "is this 802.11", not a name pattern like wlp*.
|
||||
wifi=''
|
||||
for d in /sys/class/net/*; do
|
||||
[ -e "$d/phy80211" ] || continue
|
||||
i=$(basename "$d")
|
||||
wifi="$wifi $i"
|
||||
m=$(basename "$(readlink -f "$d/device/driver" 2>/dev/null || echo unknown)")
|
||||
state=$(cat "$d/operstate" 2>/dev/null || echo unknown)
|
||||
printf 'IFACE|%s\n' "$i"
|
||||
printf 'DETAIL|%s driver=%s operstate=%s\n' "$i" "$m" "$state"
|
||||
# LIVE means "not already down" — the link-down task keys off this, so a node whose
|
||||
# radio is already down reports no change.
|
||||
[ "$state" = down ] || printf 'IFACE_LIVE|%s\n' "$i"
|
||||
ip -o addr show dev "$i" 2>/dev/null | awk '{print "IFACE_ADDR|" $4}'
|
||||
[ "$m" = unknown ] || printf 'MODULE|%s\n' "$m"
|
||||
done
|
||||
for m in $(printf '%s\n' $wifi | while read -r i; do
|
||||
[ -n "$i" ] && basename "$(readlink -f "/sys/class/net/$i/device/driver" 2>/dev/null || echo unknown)"
|
||||
done | sort -u); do
|
||||
[ "$m" = unknown ] && continue
|
||||
[ -d "/sys/module/$m" ] || continue
|
||||
printf 'MODULE_LOADED|%s\n' "$m"
|
||||
# Holders must be unloaded FIRST: `modprobe -r iwlwifi` fails while iwlmvm still
|
||||
# holds it. Read from sysfs rather than assumed, so a different wifi driver stack
|
||||
# (ath*, rtw*, mt76*) works without editing this role.
|
||||
for h in $(ls "/sys/module/$m/holders" 2>/dev/null); do printf 'MODULE_HOLDER|%s\n' "$h"; done
|
||||
done
|
||||
dr=$(ip -o route show default 2>/dev/null | awk '{for (i = 1; i < NF; i++) if ($i == "dev") print $(i + 1)}' | head -1)
|
||||
printf 'DEFROUTE_IFACE|%s\n' "${dr:-none}"
|
||||
if [ -n "${dr:-}" ] && [ -e "/sys/class/net/$dr" ]; then
|
||||
printf 'DEFROUTE_MODULE|%s\n' "$(basename "$(readlink -f "/sys/class/net/$dr/device/driver" 2>/dev/null || echo unknown)")"
|
||||
fi
|
||||
# Any DHCP client process, with the interfaces it was started for. Used only to stop a
|
||||
# client that holds a WIRELESS interface and nothing else.
|
||||
for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do
|
||||
printf 'DHCPCLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)"
|
||||
done
|
||||
exit 0
|
||||
register: node_baseline_wifi_scan
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Interpret the wireless survey
|
||||
ansible.builtin.set_fact:
|
||||
node_baseline_wifi_ifaces: "{{ scan | regex_findall('(?m)^IFACE\\|(.+)$') }}"
|
||||
node_baseline_wifi_live: "{{ scan | regex_findall('(?m)^IFACE_LIVE\\|(.+)$') }}"
|
||||
node_baseline_wifi_addrs: "{{ scan | regex_findall('(?m)^IFACE_ADDR\\|(\\S+)') | map('regex_replace', '/.*$', '') | list }}"
|
||||
node_baseline_wifi_modules: "{{ scan | regex_findall('(?m)^MODULE\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_modules_loaded: "{{ scan | regex_findall('(?m)^MODULE_LOADED\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_module_holders: "{{ scan | regex_findall('(?m)^MODULE_HOLDER\\|(.+)$') | unique }}"
|
||||
node_baseline_wifi_detail: "{{ scan | regex_findall('(?m)^DETAIL\\|(.+)$') }}"
|
||||
node_baseline_wifi_dhcp_clients: "{{ scan | regex_findall('(?m)^DHCPCLIENT\\|(.+)$') }}"
|
||||
node_baseline_defroute_iface: "{{ (scan | regex_findall('(?m)^DEFROUTE_IFACE\\|(.+)$') | first) | default('none', true) }}"
|
||||
node_baseline_defroute_module: "{{ (scan | regex_findall('(?m)^DEFROUTE_MODULE\\|(.+)$') | first) | default('unknown', true) }}"
|
||||
vars:
|
||||
scan: "{{ node_baseline_wifi_scan.stdout | default('') }}"
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Report this node's wireless hardware
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
wireless devices = {{ node_baseline_wifi_detail | default([]) | join('; ') | default('(none)', true) }}
|
||||
| driver modules = {{ node_baseline_wifi_modules | default([]) | join(', ') | default('(none)', true) }}
|
||||
(loaded: {{ node_baseline_wifi_modules_loaded | default([]) | join(', ') | default('(none)', true) }},
|
||||
held by: {{ node_baseline_wifi_module_holders | default([]) | join(', ') | default('(none)', true) }})
|
||||
| default route via {{ node_baseline_defroute_iface | default('?') }}
|
||||
(driver {{ node_baseline_defroute_module | default('?') }})
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# The one way this block could do real harm is by removing the interface it is being run
|
||||
# over. Checked from the node's own facts, per node, every run — not assumed from the fact
|
||||
# that prod "should" be wired.
|
||||
- name: Refuse to disable wireless if the path to this node runs OVER the wireless interface
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- node_baseline_defroute_iface not in node_baseline_wifi_ifaces
|
||||
- node_baseline_defroute_module not in node_baseline_wifi_modules
|
||||
- (ansible_host | default(inventory_hostname)) not in node_baseline_wifi_addrs
|
||||
fail_msg: >-
|
||||
REFUSING to disable wireless on this node: the connection to it appears to BE the
|
||||
wireless path. ansible_host={{ ansible_host | default(inventory_hostname) }}, wireless
|
||||
interfaces={{ node_baseline_wifi_ifaces | join(', ') }} carrying
|
||||
{{ node_baseline_wifi_addrs | join(', ') | default('no addresses', true) }}, default route
|
||||
via {{ node_baseline_defroute_iface }} (driver {{ node_baseline_defroute_module }}).
|
||||
Disabling the driver here would drop this node off the network with no remote way back.
|
||||
Give the node a wired address, point the inventory at it, and re-run.
|
||||
quiet: true
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_ifaces | length > 0
|
||||
tags: [wireless]
|
||||
|
||||
# THE PERSISTENT HALF. This file is the whole reboot story: kmod consults /etc/modprobe.d on
|
||||
# every load attempt, so from here on the driver cannot be loaded by udev at boot, as a
|
||||
# dependency, or by hand. Written ONLY when the node actually has a wireless driver, so a
|
||||
# node without wifi hardware (every dev VM) gets no file and no change at all.
|
||||
# `update-initramfs` is deliberately NOT run: no wifi module is loaded from the initramfs on
|
||||
# these nodes (root is a local SATA/NVMe device), the driver is autoloaded by udev after
|
||||
# switch-root, and re-generating an initramfs on a prod node is a bigger risk than the one it
|
||||
# would cover.
|
||||
- name: Block the wireless driver module(s) from ever loading again
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ node_baseline_wireless_modprobe_file }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
# Managed by the Ansible node_baseline role (tags: wireless) — do not edit by hand.
|
||||
#
|
||||
# WHY THIS NODE HAS NO WIRELESS. A k3s server gets exactly ONE path onto the network.
|
||||
# A second, DHCP-managed interface on the same LAN gives the node a second default
|
||||
# route, a second address nothing in the cluster's config knows about, and a DHCP
|
||||
# client whose resolv.conf hook silently overwrites the resolvers Ansible manages
|
||||
# (that is what happened on k3s-prod-3, 2026-09-04). The link is also brought down and
|
||||
# the module unloaded at the same time, so no reboot is needed for this to take effect.
|
||||
#
|
||||
# `blacklist` stops udev's alias-driven autoload; the `install ... /bin/false` line is
|
||||
# what also refuses a dependency-triggered or hand-run `modprobe`. Both are needed.
|
||||
#
|
||||
# TO RE-ENABLE WIRELESS ON THIS NODE: delete this file and `modprobe` the module(s)
|
||||
# below (or reboot). To stop Ansible putting it back, set
|
||||
# node_baseline_disable_wireless: false for this host first — otherwise the next
|
||||
# node-baseline run recreates it. Full rationale: roles/node_baseline/tasks/main.yml,
|
||||
# the `wireless` block, and infra/PROD-PROVISIONING.md §4.
|
||||
{% for module in node_baseline_wifi_modules %}
|
||||
blacklist {{ module }}
|
||||
install {{ module }} /bin/false
|
||||
{% endfor %}
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_modules | length > 0
|
||||
tags: [wireless]
|
||||
|
||||
# THE RUNTIME HALF — everything below makes the change true NOW, so the node does not have to
|
||||
# be rebooted to stop carrying a second address (this role never reboots: §0b "SECOND FAULT").
|
||||
#
|
||||
# Stop a DHCP client that is holding a wireless interface first, so it releases its lease
|
||||
# instead of being left holding a device that is about to disappear. HEAVILY GUARDED: it runs
|
||||
# only for clients whose command line names a wireless interface and names NO non-wireless
|
||||
# interface, because `dhcpcd -x` against a shared master process would also drop a wired
|
||||
# lease. On these prod nodes eno1 is a static ifupdown address with no DHCP client at all, so
|
||||
# this task has nothing to do; it exists so the ordering is correct on a node where wifi is
|
||||
# still actively leasing.
|
||||
- name: Stop the dhcpcd instance that holds a wireless interface (and nothing else)
|
||||
ansible.builtin.command:
|
||||
argv: [dhcpcd, --exit, "{{ item }}"]
|
||||
loop: "{{ node_baseline_wifi_ifaces }}"
|
||||
register: node_baseline_wifi_dhcpcd_exit
|
||||
changed_when: rc == 0
|
||||
failed_when: false
|
||||
when:
|
||||
- node_baseline_disable_wireless | bool
|
||||
- node_baseline_wifi_dhcp_wifi_only | length > 0
|
||||
- node_baseline_wifi_dhcp_wifi_only | select('search', 'dhcpcd') | list | length > 0
|
||||
vars:
|
||||
# A client is "wireless-only" when its argv mentions a wireless interface and mentions no
|
||||
# other interface this node has. Empty interface sets are turned into a pattern that
|
||||
# cannot match, so an odd node fails towards doing nothing rather than towards guessing.
|
||||
node_baseline_non_wifi_ifaces: "{{ ansible_facts['interfaces'] | default([]) | difference(node_baseline_wifi_ifaces) | difference(['lo']) }}"
|
||||
node_baseline_wifi_dhcp_wifi_only: >-
|
||||
{{ node_baseline_wifi_dhcp_clients
|
||||
| select('search', '\b(' ~ ((node_baseline_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b')
|
||||
| reject('search', '\b(' ~ ((node_baseline_non_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b')
|
||||
| list }}
|
||||
tags: [wireless]
|
||||
|
||||
- name: Take each wireless link down now
|
||||
ansible.builtin.command:
|
||||
argv: [ip, link, set, dev, "{{ item }}", down]
|
||||
loop: "{{ node_baseline_wifi_live }}"
|
||||
changed_when: true
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# Unload the driver so the netdev is gone immediately — holders (iwlmvm on these nodes) first,
|
||||
# then the driver itself, because `modprobe -r` refuses a module that is still held.
|
||||
# NOT fatal on failure: the persistent block above is already in place, so a module that
|
||||
# cannot be unloaded right now is gone at the next boot regardless. The report below says so
|
||||
# out loud rather than letting a green play imply the radio is already off.
|
||||
- name: Unload the wireless driver module(s) now (no reboot needed)
|
||||
community.general.modprobe:
|
||||
name: "{{ item }}"
|
||||
state: absent
|
||||
loop: "{{ node_baseline_wifi_module_holders + node_baseline_wifi_modules_loaded }}"
|
||||
register: node_baseline_wifi_unload
|
||||
failed_when: false
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
- name: Report the wireless state this node is left in
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ 'No 802.11 device on this node — nothing to disable (no modprobe.d file written).'
|
||||
if (node_baseline_wifi_ifaces | length) == 0
|
||||
else ((('WIRELESS DISABLED: ' ~ (node_baseline_wifi_modules | join(', ')) ~ ' blocked in '
|
||||
~ node_baseline_wireless_modprobe_file ~ ' (survives reboot) and unloaded now; '
|
||||
~ (node_baseline_wifi_ifaces | join(', ')) ~ ' is down and the netdev is gone.')
|
||||
if (failed_unloads | length) == 0
|
||||
else ('WIRELESS BLOCKED FOR THE NEXT BOOT BUT STILL LOADED NOW: could not unload '
|
||||
~ (failed_unloads | join(', ')) ~ '. ' ~ node_baseline_wireless_modprobe_file
|
||||
~ ' is in place, so the driver will not load again after a reboot, but the '
|
||||
~ 'radio is live until then — check `ip link` and whether something still '
|
||||
~ 'holds the device.'))) }}
|
||||
vars:
|
||||
failed_unloads: >-
|
||||
{{ (node_baseline_wifi_unload.results | default([]))
|
||||
| selectattr('failed', 'defined') | selectattr('failed') | map(attribute='item') | list }}
|
||||
when: node_baseline_disable_wireless | bool
|
||||
tags: [wireless]
|
||||
|
||||
# =====================================================================================
|
||||
# DATA DISKS — tag `disks`, in tasks/disks.yml
|
||||
# =====================================================================================
|
||||
@@ -377,6 +681,11 @@
|
||||
# node and the matching seam is written. Adding a third mechanism means adding a branch
|
||||
# here, not rewriting the role.
|
||||
#
|
||||
# AND DETECTION IS ONLY AS GOOD AS THE SET OF OWNERS IT KNOWS ABOUT. It originally looked for
|
||||
# resolvconf/openresolv and NetworkManager, missed dhcpcd, and therefore reported a green,
|
||||
# `changed` run on k3s-prod-3 whose resolvers a wifi lease reverted three minutes later. dhcpcd
|
||||
# is now part of that set — see the detection task below and the `contended` failure.
|
||||
#
|
||||
# All of it is `dns`-tagged, so `--tags packages` (the prod package-floor escape hatch
|
||||
# documented at the top of this file) still installs nothing but packages.
|
||||
|
||||
@@ -396,6 +705,55 @@
|
||||
check_mode: false
|
||||
tags: [dns]
|
||||
|
||||
# THE dhcpcd BLIND SPOT, CLOSED (k3s-prod-3, 2026-09-04). dhcpcd's `20-resolv.conf` hook
|
||||
# rewrites /etc/resolv.conf on every lease, renewal and carrier change, and it is a THIRD
|
||||
# way to own that file — neither resolvconf nor NetworkManager. On node 3 it silently reverted
|
||||
# this role's write three minutes later, and because nothing below looked for it the role had
|
||||
# already reported `changed` and green: the precise failure the `contended` branch exists to
|
||||
# make impossible.
|
||||
#
|
||||
# WHY THIS LOOKS FOR A LIVE CLIENT AND NOT FOR AN INSTALLED PACKAGE. `dhcpcd-base` is
|
||||
# installed on all three prod nodes and always will be — it is the DHCP client ifupdown
|
||||
# reaches for — and there is NO dhcpcd.service on them at all (`systemctl is-active dhcpcd` =
|
||||
# not-found; it is spawned per-interface by ifupdown). Keying off the package, or off a unit
|
||||
# that does not exist, would put every prod node permanently in `contended` and leave the DNS
|
||||
# block unable to configure a node it is perfectly safe to configure. What actually matters is
|
||||
# whether a DHCP client CAN still write the file, so all three real signals are checked:
|
||||
# CLIENT| a dhcpcd/dhclient process running right now (with the argv that started it);
|
||||
# UNIT| a dhcpcd/dhclient unit that IS active or IS enabled, on a distro that has one;
|
||||
# STANZA| an UNCOMMENTED `iface <x> inet[6] dhcp` in /etc/network/interfaces{,.d} — the
|
||||
# configuration that makes ifupdown spawn a client at the next boot or `ifup`, even
|
||||
# when none is running at this instant.
|
||||
# Any one of them means "another manager is in play" and the play stops below. None of them
|
||||
# means no client can run, and the file is genuinely ours to own.
|
||||
- name: Determine whether a DHCP client (dhcpcd/dhclient) can rewrite /etc/resolv.conf
|
||||
ansible.builtin.shell:
|
||||
executable: /bin/sh
|
||||
cmd: |
|
||||
set -u
|
||||
for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do
|
||||
printf 'CLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)"
|
||||
done
|
||||
for u in dhcpcd dhcpcd5 dhclient isc-dhcp-client; do
|
||||
a=$(systemctl is-active "$u" 2>/dev/null || true)
|
||||
e=$(systemctl is-enabled "$u" 2>/dev/null || true)
|
||||
# Compared exactly, never with a substring match: `is-active` answers "inactive",
|
||||
# which contains the word "active".
|
||||
if [ "$a" = active ] || [ "$e" = enabled ] || [ "$e" = enabled-runtime ]; then
|
||||
printf 'UNIT|%s active=%s enabled=%s\n' "$u" "$a" "$e"
|
||||
fi
|
||||
done
|
||||
# `^[[:space:]]*iface` cannot match a commented-out stanza, which is how the wireless
|
||||
# stanza is left on these nodes.
|
||||
grep -rhE '^[[:space:]]*iface[[:space:]]+[^[:space:]]+[[:space:]]+inet6?[[:space:]]+dhcp' \
|
||||
/etc/network/interfaces /etc/network/interfaces.d/ 2>/dev/null \
|
||||
| sed -e 's/^[[:space:]]*/STANZA|/'
|
||||
exit 0
|
||||
register: node_baseline_dhcp_client_scan
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
tags: [dns]
|
||||
|
||||
# follow: false — we need to know whether /etc/resolv.conf IS a symlink and where it
|
||||
# points, which is the strongest single signal of who owns the file.
|
||||
- name: Inspect /etc/resolv.conf itself (symlink or real file, and its target)
|
||||
@@ -426,9 +784,10 @@
|
||||
# resolved's loopback stub (127.0.0.53 / 127.0.0.54). Writing a resolved drop-in is
|
||||
# then the ONLY thing that changes real behaviour; writing resolv.conf would be
|
||||
# overwritten by resolved (or, worse, land in /run and vanish at reboot).
|
||||
# contended — nobody's stub, but resolvconf or NetworkManager is present/running and
|
||||
# would regenerate the file underneath us. We do NOT write. The role fails loudly
|
||||
# below rather than leaving an operator believing a change stuck when it did not.
|
||||
# contended — nobody's stub, but resolvconf, NetworkManager or a DHCP client (dhcpcd /
|
||||
# dhclient) is present/running/configured and would regenerate the file underneath us.
|
||||
# We do NOT write. The role fails loudly below rather than leaving an operator
|
||||
# believing a change stuck when it did not.
|
||||
# resolv-conf-file — a plain, directly-managed /etc/resolv.conf (prod's shape today).
|
||||
# Safe to own outright.
|
||||
# resolved running but NOT owning resolv.conf correctly falls through to the file branch:
|
||||
@@ -438,13 +797,21 @@
|
||||
node_baseline_dns_mechanism: >-
|
||||
{{
|
||||
'systemd-resolved' if (node_baseline_resolved_running and node_baseline_resolved_owns_resolv)
|
||||
else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present)
|
||||
else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present
|
||||
or node_baseline_dhcp_client_live)
|
||||
else 'resolv-conf-file')
|
||||
}}
|
||||
# Kept as facts (not just task vars) so the failure message, the report and the
|
||||
# reclaim note below all quote the SAME evidence this decision was made on.
|
||||
node_baseline_dhcp_client_evidence: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) | regex_findall('(?m)^(?:CLIENT|UNIT|STANZA)\\|(.+)$') }}"
|
||||
# A file dhcpcd WROTE, versus a dhcpcd that can still write it — two different facts, and
|
||||
# only the second one is contention. See the reclaim note below.
|
||||
node_baseline_resolv_dhcpcd_written: "{{ 'Generated by dhcpcd' in (node_baseline_resolv_slurp.content | default('') | b64decode) }}"
|
||||
vars:
|
||||
node_baseline_resolved_running: "{{ (node_baseline_resolved_active.stdout | default('') | trim) == 'active' }}"
|
||||
node_baseline_nm_running: "{{ (node_baseline_nm_active.stdout | default('') | trim) == 'active' }}"
|
||||
node_baseline_resolvconf_present: "{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}"
|
||||
node_baseline_dhcp_client_live: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) is search('(?m)^(?:CLIENT|UNIT|STANZA)\\|') }}"
|
||||
node_baseline_resolv_text: "{{ (node_baseline_resolv_slurp.content | default('') | b64decode) }}"
|
||||
node_baseline_resolv_ns: "{{ node_baseline_resolv_text | regex_findall('(?m)^\\s*nameserver\\s+(\\S+)') }}"
|
||||
node_baseline_resolved_owns_resolv: >-
|
||||
@@ -461,7 +828,9 @@
|
||||
NetworkManager is-active={{ node_baseline_nm_active.stdout | default('') | trim }},
|
||||
/etc/resolv.conf islnk={{ node_baseline_resolv_stat.stat.islnk | default(false) }}
|
||||
target={{ node_baseline_resolv_stat.stat.lnk_source | default('(not a symlink)') }},
|
||||
resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }})
|
||||
resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }},
|
||||
DHCP client evidence={{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }},
|
||||
resolv.conf written by dhcpcd={{ node_baseline_resolv_dhcpcd_written | default(false) }})
|
||||
tags: [dns]
|
||||
|
||||
# Same shape, and the same reason, as the data-disk assert above: this environment's
|
||||
@@ -494,16 +863,45 @@
|
||||
- name: Refuse to manage /etc/resolv.conf while another resolver manager owns it
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
/etc/resolv.conf on this node is not the systemd-resolved stub, but
|
||||
resolvconf/openresolv (/etc/resolvconf/resolv.conf.d present:
|
||||
{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) or NetworkManager
|
||||
(is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}) is in play and
|
||||
would regenerate the file underneath this role. Writing it here would look like it
|
||||
worked and silently revert. Add the matching branch to roles/node_baseline (the
|
||||
`dns`-tagged block) for whichever manager this node really uses — do not force it.
|
||||
/etc/resolv.conf on this node is not the systemd-resolved stub, but something else would
|
||||
regenerate the file underneath this role: resolvconf/openresolv
|
||||
(/etc/resolvconf/resolv.conf.d present:
|
||||
{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}), NetworkManager
|
||||
(is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}), or a DHCP client
|
||||
that can still write it — dhcpcd/dhclient evidence:
|
||||
{{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}.
|
||||
Writing it here would look like it worked and silently revert — which is exactly what
|
||||
happened on k3s-prod-3 on 2026-09-04, where a wifi lease's dhcpcd hook reverted this
|
||||
role's resolvers three minutes after a green run. If the evidence above is a DHCP client
|
||||
on a WIRELESS interface, the fix is the `wireless` block earlier in this role (a cluster
|
||||
node gets one network path) — run it, then re-run this. If it is a DHCP client on the
|
||||
node's real interface, or another manager entirely, add the matching branch to
|
||||
roles/node_baseline for the manager this node really uses — do not force it.
|
||||
when: node_baseline_dns_mechanism == 'contended'
|
||||
tags: [dns]
|
||||
|
||||
# THE CORPSE CASE, and why it is NOT a hole in the fail-closed rule above. After the
|
||||
# `wireless` block disables the radio, k3s-prod-3 is left with a /etc/resolv.conf that dhcpcd
|
||||
# WROTE (its "# Generated by dhcpcd from wlp1s0.dhcp..." header, the gateway as sole IPv4
|
||||
# resolver, two Google IPv6 servers) and NO dhcpcd that can ever run again. `contended` means
|
||||
# "another manager is live and will undo this write"; a file left behind by a client that no
|
||||
# longer exists is not a manager, it is the damage — and refusing to write here would leave the
|
||||
# node permanently mis-resolved, which is the very outcome the operator reported. So the file
|
||||
# branch below reclaims it, and says so.
|
||||
- name: Note that a dhcpcd-written /etc/resolv.conf is being reclaimed
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
/etc/resolv.conf on this node was written by dhcpcd, but no DHCP client can write it any
|
||||
more (no process, no active/enabled unit, no `inet dhcp` stanza). Taking ownership and
|
||||
replacing it with the declared upstreams
|
||||
({{ node_baseline_dns_servers | join(', ') }}). If a DHCP client ever comes back on this
|
||||
node, the `contended` check above will stop the play instead of letting it revert
|
||||
silently.
|
||||
when:
|
||||
- node_baseline_dns_mechanism == 'resolv-conf-file'
|
||||
- node_baseline_resolv_dhcpcd_written | default(false)
|
||||
tags: [dns]
|
||||
|
||||
# --- Mechanism A: systemd-resolved (dev's shape today) -------------------------------
|
||||
# A drop-in, never an edit of /etc/systemd/resolved.conf, so a package upgrade of that file
|
||||
# never conflicts. `DNS=` is one ORDERED list inside a single resolved scope: resolved uses
|
||||
|
||||
@@ -29,9 +29,9 @@ metadata:
|
||||
data:
|
||||
# Filename `home.server` -> imported as a standalone server block for the whole zone.
|
||||
home.server: |
|
||||
home.keithjackson.dev:53 {
|
||||
${BASE_DOMAIN}:53 {
|
||||
errors
|
||||
# Answer EVERY name in the zone (apex + all `<svc>.home.keithjackson.dev`) with the Gateway
|
||||
# Answer EVERY name in the zone (apex + all `<svc>.${BASE_DOMAIN}`) with the Gateway
|
||||
# VIP. The template plugin's default match is `.*`, so this covers current and FUTURE app
|
||||
# hostnames without editing this file per service. `{{ .Name }}` echoes the queried FQDN.
|
||||
template IN A {
|
||||
|
||||
@@ -125,7 +125,7 @@ spec:
|
||||
- --http-address=0.0.0.0:4180
|
||||
- --reverse-proxy=true
|
||||
- --provider=oidc
|
||||
- --oidc-issuer-url=https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
- --oidc-issuer-url=https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
# Gate the co-located Crafty UI. Crafty serves HTTPS with a SELF-SIGNED cert, so skip verify
|
||||
# on this in-pod (127.0.0.1) hop; the browser-facing TLS is the trusted wildcard at the Gateway.
|
||||
- --upstream=https://127.0.0.1:8443
|
||||
@@ -138,9 +138,9 @@ spec:
|
||||
- --allowed-group=platform-admins
|
||||
# Single provider => 302 straight to Keycloak, no oauth2-proxy interstitial button.
|
||||
- --skip-provider-button=true
|
||||
- --redirect-url=https://crafty.home.keithjackson.dev/oauth2/callback
|
||||
- --cookie-domain=.home.keithjackson.dev
|
||||
- --whitelist-domain=.home.keithjackson.dev
|
||||
- --redirect-url=https://crafty.${BASE_DOMAIN}/oauth2/callback
|
||||
- --cookie-domain=.${BASE_DOMAIN}
|
||||
- --whitelist-domain=.${BASE_DOMAIN}
|
||||
- --cookie-secure=true
|
||||
- --pass-access-token=true
|
||||
- --set-xauthrequest=true
|
||||
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- crafty.home.keithjackson.dev
|
||||
- crafty.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -42,7 +42,7 @@ metadata:
|
||||
lbipam.cilium.io/sharing-key: "crafty"
|
||||
lbipam.cilium.io/ips: "${CRAFTY_L4_VIP}"
|
||||
# external-dns publishes A crafty-mc.home.keithjackson.dev -> the pinned VIP (Java server target).
|
||||
external-dns.alpha.kubernetes.io/hostname: "crafty-mc.home.keithjackson.dev"
|
||||
external-dns.alpha.kubernetes.io/hostname: "crafty-mc.${BASE_DOMAIN}"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
|
||||
@@ -29,15 +29,46 @@ spec:
|
||||
sources:
|
||||
- gateway-httproute
|
||||
- service
|
||||
# domain-filter MUST be the actual Cloudflare ZONE (keithjackson.dev), not the platform subdomain
|
||||
# (home.keithjackson.dev) — ExternalDNS filters the account's zones by this value, and a subdomain
|
||||
# filter drops the parent zone → "no hosted zone matching record DNS Name" and nothing is created.
|
||||
# Scope stays effectively home.keithjackson.dev because that's the only suffix services request;
|
||||
# policy=sync + registry=txt only ever touch ExternalDNS-OWNED records, never the pre-existing
|
||||
# wildcard or other keithjackson.dev records. (docs/09; resolves the WS-NET TODO below.)
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# CROSS-CLUSTER DNS OWNERSHIP. Read before changing either knob below.
|
||||
#
|
||||
# On 2026-09-04 prod's external-dns DELETED EVERY ONE of dev's public records. Both clusters ran
|
||||
# `policy: sync` against the SAME Cloudflare zone, served the SAME hostnames, and — because
|
||||
# NEITHER set txtOwnerId — both inherited the chart default owner id `default`. Prod therefore
|
||||
# read dev's TXT registry entries as its own, found no local resource claiming those names, and
|
||||
# garbage-collected them. `policy: sync` is correct and stays; what was missing was identity.
|
||||
# Full write-up: infra/PROD-PROVISIONING.md §10.
|
||||
#
|
||||
# THE TWO GUARDS THAT ACTUALLY PREVENT A RECURRENCE — note that neither is domainFilters:
|
||||
# 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns.
|
||||
# 2. DISJOINT HOSTNAME SETS — the two environments serve different suffixes, so they never
|
||||
# describe the same record in the first place. dev = *.dev.home.keithjackson.dev,
|
||||
# prod = *.home.keithjackson.dev (prod IS the apex; operator decision 2026-09-04).
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Which names this instance may manage at all. BOTH ENVIRONMENTS RENDER THE CLOUDFLARE **ZONE**
|
||||
# `keithjackson.dev` HERE, and this is NOT an oversight to be tightened.
|
||||
#
|
||||
# FINDING F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for ZONE
|
||||
# SELECTION as well as record matching, so a filter that is a STRICT SUBDOMAIN of the hosted zone
|
||||
# matches NO zone — external-dns generates the record and then skips it with "no hosted zone
|
||||
# matching record DNS Name", publishing NOTHING. This has been "improved" and reverted more than
|
||||
# once (an agent narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in
|
||||
# review). It is not defence in depth; it is a total outage of DNS publication. Isolation comes
|
||||
# from the two guards above, which a domain filter cannot provide anyway — both environments live
|
||||
# in the ONE zone regardless of what this says. See clusters/base/cluster-vars.yaml.
|
||||
domainFilters:
|
||||
- "${DNS_ZONE}"
|
||||
policy: sync # create + update + delete records as services come and go
|
||||
- "${EXTERNALDNS_DOMAIN_FILTER}"
|
||||
# The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence
|
||||
# caused the incident. dev renders `dev`, prod renders `prod`, and the BASE default is a
|
||||
# non-colliding sentinel so a new environment that forgets the knob owns nothing and can delete
|
||||
# nothing. NOTHING renders `default` any more (the external-dns chart default, and the value both
|
||||
# clusters silently shared): dev held it only because its live apex records were registered under
|
||||
# it, and dev's move to *.dev.home.keithjackson.dev retired that constraint along with those
|
||||
# records. Prod must NOT be given `default` even though it inherits the apex NAMES — that would
|
||||
# hand prod ownership of dev's old apex records and let `policy: sync` below delete them.
|
||||
txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}"
|
||||
policy: sync # create + update + delete records as services come and go — but ONLY
|
||||
# records this owner id owns AND this domainFilter admits (see above).
|
||||
# Cloudflare API token: SOPS-encrypted secret, referenced BY NAME only (OQ-4, ADR-0007).
|
||||
env:
|
||||
- name: CF_API_TOKEN
|
||||
|
||||
@@ -13,7 +13,7 @@ spec:
|
||||
- name: https
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: "*.home.keithjackson.dev"
|
||||
hostname: "*.${BASE_DOMAIN}"
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
- name: http-redirect
|
||||
protocol: HTTP
|
||||
port: 80
|
||||
hostname: "*.home.keithjackson.dev"
|
||||
hostname: "*.${BASE_DOMAIN}"
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Selector
|
||||
@@ -46,7 +46,7 @@ spec:
|
||||
- name: https-apex
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: "home.keithjackson.dev"
|
||||
hostname: "${BASE_DOMAIN}"
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
- name: http-apex-redirect
|
||||
protocol: HTTP
|
||||
port: 80
|
||||
hostname: "home.keithjackson.dev"
|
||||
hostname: "${BASE_DOMAIN}"
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Selector
|
||||
|
||||
@@ -10,8 +10,8 @@ metadata:
|
||||
spec:
|
||||
secretName: wildcard-home-tls # consumed by the Gateway HTTPS listener below
|
||||
dnsNames:
|
||||
- "*.home.keithjackson.dev"
|
||||
- "home.keithjackson.dev" # apex for the C2 UI (docs/09)
|
||||
- "*.${BASE_DOMAIN}"
|
||||
- "${BASE_DOMAIN}" # apex for the C2 UI (docs/09)
|
||||
issuerRef:
|
||||
kind: ClusterIssuer
|
||||
# ${ACME_CLUSTER_ISSUER}: dev -> letsencrypt-staging, prod -> letsencrypt-prod (cluster-vars).
|
||||
|
||||
@@ -24,7 +24,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- headlamp.home.keithjackson.dev
|
||||
- headlamp.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -36,63 +36,63 @@ data:
|
||||
---
|
||||
- Platform:
|
||||
- Headlamp:
|
||||
href: https://headlamp.home.keithjackson.dev
|
||||
href: https://headlamp.${BASE_DOMAIN}
|
||||
description: Kubernetes dashboard (per-user OIDC RBAC)
|
||||
icon: mdi-kubernetes
|
||||
- Grafana:
|
||||
href: https://grafana.home.keithjackson.dev
|
||||
href: https://grafana.${BASE_DOMAIN}
|
||||
description: Metrics, logs & dashboards (fronts Prometheus/Loki/Alertmanager)
|
||||
icon: mdi-chart-areaspline
|
||||
- Semaphore:
|
||||
href: https://semaphore.home.keithjackson.dev
|
||||
href: https://semaphore.${BASE_DOMAIN}
|
||||
description: Ansible automation & CI
|
||||
icon: mdi-cog-play
|
||||
- Gitea:
|
||||
href: https://gitea.home.keithjackson.dev
|
||||
href: https://gitea.${BASE_DOMAIN}
|
||||
description: Git hosting & Actions (GitOps source of truth)
|
||||
icon: mdi-git
|
||||
- Keycloak:
|
||||
href: https://keycloak.home.keithjackson.dev
|
||||
href: https://keycloak.${BASE_DOMAIN}
|
||||
description: Identity provider (SSO / OIDC)
|
||||
icon: mdi-account-key
|
||||
|
||||
- Media:
|
||||
- Plex:
|
||||
href: https://plex.home.keithjackson.dev
|
||||
href: https://plex.${BASE_DOMAIN}
|
||||
description: Media streaming
|
||||
icon: mdi-plex
|
||||
- Seerr:
|
||||
href: https://seerr.home.keithjackson.dev
|
||||
href: https://seerr.${BASE_DOMAIN}
|
||||
description: Media requests
|
||||
icon: mdi-play-box-multiple
|
||||
- Sonarr:
|
||||
href: https://sonarr.home.keithjackson.dev
|
||||
href: https://sonarr.${BASE_DOMAIN}
|
||||
description: TV library manager
|
||||
icon: mdi-television-classic
|
||||
- Radarr:
|
||||
href: https://radarr.home.keithjackson.dev
|
||||
href: https://radarr.${BASE_DOMAIN}
|
||||
description: Movie library manager
|
||||
icon: mdi-movie-open
|
||||
- Jackett:
|
||||
href: https://jackett.home.keithjackson.dev
|
||||
href: https://jackett.${BASE_DOMAIN}
|
||||
description: Indexer proxy
|
||||
icon: mdi-magnify
|
||||
- qBittorrent:
|
||||
href: https://qbittorrent.home.keithjackson.dev
|
||||
href: https://qbittorrent.${BASE_DOMAIN}
|
||||
description: Download client (VPN egress)
|
||||
icon: mdi-download-network
|
||||
|
||||
- Network & Infra:
|
||||
- Pi-hole:
|
||||
href: https://pihole.home.keithjackson.dev
|
||||
href: https://pihole.${BASE_DOMAIN}
|
||||
description: Network-wide DNS & ad-blocking
|
||||
icon: mdi-shield-check
|
||||
- UniFi:
|
||||
href: https://unifi.home.keithjackson.dev:11443
|
||||
href: https://unifi.${BASE_DOMAIN}:11443
|
||||
description: Network controller (UOS)
|
||||
icon: mdi-router-network
|
||||
- Nextcloud:
|
||||
href: https://nextcloud.home.keithjackson.dev
|
||||
href: https://nextcloud.${BASE_DOMAIN}
|
||||
description: Files, calendar & contacts
|
||||
icon: mdi-cloud
|
||||
|
||||
@@ -104,7 +104,7 @@ data:
|
||||
href: https://gitea.keithjackson.dev
|
||||
- Auth portal:
|
||||
- abbr: SSO
|
||||
href: https://auth.home.keithjackson.dev
|
||||
href: https://auth.${BASE_DOMAIN}
|
||||
- Reference:
|
||||
- Homepage docs:
|
||||
- abbr: HP
|
||||
|
||||
@@ -54,7 +54,7 @@ spec:
|
||||
# REQUIRED by Homepage >= 1.0 — requests whose Host header is not in this list get 403.
|
||||
# The public apex host (served via the Gateway) + the pod IP (for kubelet probes).
|
||||
- name: HOMEPAGE_ALLOWED_HOSTS
|
||||
value: "home.keithjackson.dev,$(MY_POD_IP):3000"
|
||||
value: "${BASE_DOMAIN},$(MY_POD_IP):3000"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
|
||||
@@ -28,7 +28,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https-apex # the apex listener (wildcard `https` won't match the bare apex)
|
||||
hostnames:
|
||||
- home.keithjackson.dev
|
||||
- ${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- keycloak.home.keithjackson.dev
|
||||
- keycloak.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
poolMaxSize: 20
|
||||
# --- Exposure: TLS at the Gateway; Keycloak serves HTTP and trusts the proxy headers. ---
|
||||
hostname:
|
||||
hostname: https://keycloak.home.keithjackson.dev
|
||||
hostname: https://keycloak.${BASE_DOMAIN}
|
||||
http:
|
||||
httpEnabled: true # TLS is terminated at the Gateway (edge), not in Keycloak
|
||||
proxy:
|
||||
|
||||
@@ -98,9 +98,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:GRAFANA_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://grafana.home.keithjackson.dev/login/generic_oauth
|
||||
- https://grafana.${BASE_DOMAIN}/login/generic_oauth
|
||||
webOrigins:
|
||||
- https://grafana.home.keithjackson.dev
|
||||
- https://grafana.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -125,10 +125,10 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:OAUTH2_PROXY_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://*.home.keithjackson.dev/oauth2/callback
|
||||
- https://auth.home.keithjackson.dev/oauth2/callback
|
||||
- https://*.${BASE_DOMAIN}/oauth2/callback
|
||||
- https://auth.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://*.home.keithjackson.dev
|
||||
- https://*.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -209,9 +209,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:GITEA_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://gitea.home.keithjackson.dev/user/oauth2/keycloak/callback
|
||||
- https://gitea.${BASE_DOMAIN}/user/oauth2/keycloak/callback
|
||||
webOrigins:
|
||||
- https://gitea.home.keithjackson.dev
|
||||
- https://gitea.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -240,9 +240,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:NEXTCLOUD_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://nextcloud.home.keithjackson.dev/apps/user_oidc/code
|
||||
- https://nextcloud.${BASE_DOMAIN}/apps/user_oidc/code
|
||||
webOrigins:
|
||||
- https://nextcloud.home.keithjackson.dev
|
||||
- https://nextcloud.${BASE_DOMAIN}
|
||||
# Post-logout redirect allow-list (#logout "redirect failed"). On sign-out user_oidc redirects the
|
||||
# browser to Keycloak's end_session_endpoint with post_logout_redirect_uri=<Nextcloud base '/'> and
|
||||
# client_id=nextcloud but NO id_token_hint (provider sendIdTokenHint=0), so Keycloak validates the
|
||||
@@ -250,7 +250,7 @@ spec:
|
||||
# "redirect failed". The wildcard covers the base URL and any logout landing path. Keycloak stores
|
||||
# this as the client attribute `post.logout.redirect.uris` (## separates multiple values).
|
||||
attributes:
|
||||
post.logout.redirect.uris: https://nextcloud.home.keithjackson.dev/*
|
||||
post.logout.redirect.uris: https://nextcloud.${BASE_DOMAIN}/*
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -279,9 +279,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:SEMAPHORE_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://semaphore.home.keithjackson.dev/api/auth/oidc/keycloak/redirect
|
||||
- https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect
|
||||
webOrigins:
|
||||
- https://semaphore.home.keithjackson.dev
|
||||
- https://semaphore.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -318,9 +318,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:HEADLAMP_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://headlamp.home.keithjackson.dev/oidc-callback
|
||||
- https://headlamp.${BASE_DOMAIN}/oidc-callback
|
||||
webOrigins:
|
||||
- https://headlamp.home.keithjackson.dev
|
||||
- https://headlamp.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -363,9 +363,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:CRAFTY_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://crafty.home.keithjackson.dev/oauth2/callback
|
||||
- https://crafty.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://crafty.home.keithjackson.dev
|
||||
- https://crafty.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -398,9 +398,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:PIHOLE_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://pihole.home.keithjackson.dev/oauth2/callback
|
||||
- https://pihole.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://pihole.home.keithjackson.dev
|
||||
- https://pihole.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -433,9 +433,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:JACKETT_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://jackett.home.keithjackson.dev/oauth2/callback
|
||||
- https://jackett.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://jackett.home.keithjackson.dev
|
||||
- https://jackett.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -459,9 +459,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:QBITTORRENT_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://qbittorrent.home.keithjackson.dev/oauth2/callback
|
||||
- https://qbittorrent.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://qbittorrent.home.keithjackson.dev
|
||||
- https://qbittorrent.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -485,9 +485,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:RADARR_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://radarr.home.keithjackson.dev/oauth2/callback
|
||||
- https://radarr.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://radarr.home.keithjackson.dev
|
||||
- https://radarr.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
@@ -511,9 +511,9 @@ spec:
|
||||
directAccessGrantsEnabled: false
|
||||
secret: "$(env:SONARR_CLIENT_SECRET)"
|
||||
redirectUris:
|
||||
- https://sonarr.home.keithjackson.dev/oauth2/callback
|
||||
- https://sonarr.${BASE_DOMAIN}/oauth2/callback
|
||||
webOrigins:
|
||||
- https://sonarr.home.keithjackson.dev
|
||||
- https://sonarr.${BASE_DOMAIN}
|
||||
defaultClientScopes:
|
||||
- email
|
||||
- profile
|
||||
|
||||
@@ -30,7 +30,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt VM(s) failing to launch in {{ $labels.namespace }}"
|
||||
description: "One or more VirtualMachineInstances are in the Failed phase in namespace {{ $labels.namespace }} for 5m — a failed VM launch (image import, scheduling, or qemu error). Check `kubectl get vmi -n {{ $labels.namespace }}` and the virt-launcher pod events."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-launch-failed"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-launch-failed"
|
||||
# Crash-loop: the VM's virt-launcher pod is restarting repeatedly (guest kernel panic, OOM, or a
|
||||
# crashing qemu). >3 restarts in 15m is a crash-loop, not a one-off reboot.
|
||||
- alert: KubeVirtVMCrashLooping
|
||||
@@ -42,7 +42,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt VM crash-looping: {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "virt-launcher pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in 15m — the guest VM is crash-looping (panic / OOM / qemu crash)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-crashloop"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-crashloop"
|
||||
# Control plane down: no virt-controller or virt-api replica reports ready, so no VM lifecycle
|
||||
# operations can proceed. The metrics are `kubevirt_virt_{controller,api}_ready_status` (per-pod
|
||||
# 1=ready); sum()==0 means every replica is unready. The absent() clause also fires if the whole
|
||||
@@ -59,7 +59,7 @@ spec:
|
||||
annotations:
|
||||
summary: "KubeVirt control plane degraded"
|
||||
description: "The KubeVirt control plane (virt-controller/virt-api) is not ready or its metrics are absent for 10m — VM lifecycle operations (create/start/stop) will fail (docs/06)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/kubevirt-control-plane"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/kubevirt-control-plane"
|
||||
# CDI import stuck: a DataVolume import/clone still in flight after 30m blocks the VM it backs.
|
||||
# kubevirt_cdi_import_pods_high_restart counts importer pods stuck restarting; a positive value
|
||||
# sustained means an import is not progressing (metric name confirmed against CDI at bring-up).
|
||||
@@ -71,4 +71,4 @@ spec:
|
||||
annotations:
|
||||
summary: "CDI DataVolume import stuck: {{ $labels.namespace }}/{{ $labels.name }}"
|
||||
description: "A CDI DataVolume import/clone in {{ $labels.namespace }} has not completed after 30m — the VM disk it backs cannot start. Check the importer pod (often an egress/default-deny block on the image source, or a slow mirror)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cdi-import-stuck"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cdi-import-stuck"
|
||||
@@ -38,7 +38,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Node exporter down on {{ $labels.instance }}"
|
||||
description: "Prometheus cannot scrape node-exporter on {{ $labels.instance }} for 5m — the node may be down or unreachable."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-down"
|
||||
- alert: NodeMemoryPressureHigh
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
|
||||
for: 10m
|
||||
@@ -47,7 +47,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Memory pressure high on {{ $labels.instance }}"
|
||||
description: "Available memory below 10% for 10m on {{ $labels.instance }} — the ~22 GiB box is under pressure."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-memory"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-memory"
|
||||
- alert: NodeDiskFillingUp
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}
|
||||
@@ -58,7 +58,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Disk filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
||||
description: "Less than 10% free on {{ $labels.mountpoint }} of {{ $labels.instance }} for 15m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-disk"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-disk"
|
||||
- alert: NodeCPUPressureHigh
|
||||
# PSI "some" for CPU = fraction of wall-clock time at least one task was stalled waiting to
|
||||
# run. <5% is healthy; sustained >15% means bursty workloads are queueing against the vCPU
|
||||
@@ -71,7 +71,7 @@ spec:
|
||||
annotations:
|
||||
summary: "CPU pressure high on {{ $labels.instance }}"
|
||||
description: "CPU PSI (some) above 15% for 15m on {{ $labels.instance }} — tasks are being delayed waiting for CPU (bursts exceeding the node's vCPU headroom). Add vCPUs or trim CPU-heavy workloads."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu"
|
||||
- alert: NodeCPULoadHigh
|
||||
# 5-min load average per core. Sustained >1 means the run queue exceeds the core count (more
|
||||
# runnable/blocked tasks than CPUs). Core count = number of idle-mode cpu series per node.
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Load per core high on {{ $labels.instance }}"
|
||||
description: "5-min load average per core above 1.0 for 15m on {{ $labels.instance }} — the node's run queue exceeds its core count."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/node-cpu"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/node-cpu"
|
||||
|
||||
# =====================================================================================
|
||||
# Storage — Longhorn
|
||||
@@ -113,7 +113,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume degraded: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} has been degraded (replica rebuild / missing replica) for 10m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-degraded"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-degraded"
|
||||
- alert: LonghornVolumeFaulted
|
||||
expr: longhorn_volume_robustness == 3
|
||||
for: 1m
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume FAULTED: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is faulted — data at risk / offline. Immediate attention required."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-faulted"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-faulted"
|
||||
- alert: LonghornVolumeCapacityHigh
|
||||
expr: (longhorn_volume_actual_size_bytes / longhorn_volume_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
@@ -131,7 +131,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn volume near capacity: {{ $labels.volume }}"
|
||||
description: "Volume {{ $labels.volume }} is over 85% used for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity"
|
||||
- alert: LonghornNodeStorageHigh
|
||||
expr: (longhorn_node_storage_usage_bytes / longhorn_node_storage_capacity_bytes) > 0.85
|
||||
for: 30m
|
||||
@@ -140,7 +140,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Longhorn node storage near capacity: {{ $labels.node }}"
|
||||
description: "Longhorn node {{ $labels.node }} is over 85% of its storage capacity for 30m."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/longhorn-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/longhorn-capacity"
|
||||
|
||||
# =====================================================================================
|
||||
# GitOps — Flux
|
||||
@@ -169,7 +169,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux resource not ready: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has failed to reconcile (Ready=False) for 15m — GitOps drift/failure (R-GITOPS-03)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-reconcile"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-reconcile"
|
||||
- alert: FluxResourceSuspended
|
||||
expr: |
|
||||
max by (namespace, name, kind) (gotk_suspend_status) == 1
|
||||
@@ -179,7 +179,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux resource suspended: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} has been suspended for 1h — reconciliation/drift-correction is off."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/flux-suspended"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-suspended"
|
||||
|
||||
# =====================================================================================
|
||||
# Certificates — cert-manager
|
||||
@@ -205,7 +205,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate expiring soon: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 14 days."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry"
|
||||
- alert: CertificateExpiryImminent
|
||||
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) < (3 * 24 * 3600)
|
||||
for: 1h
|
||||
@@ -214,7 +214,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate expiry imminent: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in under 3 days and has not renewed."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-expiry"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-expiry"
|
||||
- alert: CertificateNotReady
|
||||
expr: certmanager_certificate_ready_status{condition="False"} == 1
|
||||
for: 1h
|
||||
@@ -223,7 +223,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Certificate not ready / issuance failing: {{ $labels.name }}"
|
||||
description: "Certificate {{ $labels.namespace }}/{{ $labels.name }} has been in a not-ready state for 1h (ACME issuance failure?)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/cert-issuance"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/cert-issuance"
|
||||
|
||||
# =====================================================================================
|
||||
# Networking — VPN / egress-gateway / denied flows (Hubble)
|
||||
@@ -255,7 +255,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Egress VPN tunnel down"
|
||||
description: "The egress-gateway VPN tunnel is down (or its metric is absent) for 5m — outbound traffic could leak or stall (leak risk, docs/07)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/vpn-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/vpn-down"
|
||||
# (ADR-0015) The former `EgressGatewayUnhealthy` pod-readiness alert was REMOVED with the gluetun
|
||||
# retirement: there is no in-cluster egress-gateway pod anymore (the tunnel is a host systemd
|
||||
# service, steering is Cilium eBPF). Tunnel + gateway-node health is covered by `VPNTunnelDown`
|
||||
@@ -270,7 +270,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Spike in policy-denied network flows"
|
||||
description: "Policy-denied flow rate is elevated for 10m — a NetworkPolicy regression or an intrusion probe (security audit, docs/07)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/denied-flows"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/denied-flows"
|
||||
|
||||
# =====================================================================================
|
||||
# Backups — Velero
|
||||
@@ -296,7 +296,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Velero backup failed: schedule {{ $labels.schedule }}"
|
||||
description: "One or more Velero backups failed in the last hour (R-STATE-03 — off-cluster DR at risk)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-failure"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-failure"
|
||||
- alert: VeleroBackupPartialFailure
|
||||
expr: increase(velero_backup_partial_failure_total[1h]) > 0
|
||||
for: 5m
|
||||
@@ -305,7 +305,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Velero backup partially failed"
|
||||
description: "A Velero backup completed with partial failures in the last hour — some resources/volumes were skipped."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-partial"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-partial"
|
||||
- alert: VeleroNoRecentBackup
|
||||
expr: (time() - velero_backup_last_successful_timestamp{schedule!=""}) > (24 * 3600)
|
||||
for: 1h
|
||||
@@ -314,7 +314,7 @@ spec:
|
||||
annotations:
|
||||
summary: "No successful Velero backup in 24h: {{ $labels.schedule }}"
|
||||
description: "Schedule {{ $labels.schedule }} has not produced a successful backup in over 24h (missed schedule)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/backup-missed"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/backup-missed"
|
||||
|
||||
# =====================================================================================
|
||||
# Identity — Keycloak (SSO is a critical dependency — docs/15)
|
||||
@@ -345,7 +345,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Keycloak IdP down"
|
||||
description: "No ready Keycloak pod for 5m — SSO is unavailable and dependent services cannot authenticate (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-down"
|
||||
- alert: KeycloakDatabaseDown
|
||||
# Keycloak's PostgreSQL runs as a CloudNativePG Cluster `keycloak-db` (pods keycloak-db-N).
|
||||
# DR-critical, Longhorn-backed. (CNPG also exports cnpg_* metrics via its PodMonitor.)
|
||||
@@ -358,7 +358,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Keycloak database unavailable"
|
||||
description: "Keycloak's PostgreSQL is not ready for 5m — the IdP will fail (docs/15)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/keycloak-db"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/keycloak-db"
|
||||
|
||||
# =====================================================================================
|
||||
# The monitor's own capacity — don't let the monitor blind itself (docs/10)
|
||||
@@ -386,7 +386,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Monitoring PVC filling up: {{ $labels.persistentvolumeclaim }}"
|
||||
description: "PVC {{ $labels.persistentvolumeclaim }} (Prometheus/Loki) is under 15% free — retention/PVC size may need raising before the monitor goes blind."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/monitoring-capacity"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/monitoring-capacity"
|
||||
- alert: PrometheusTargetsMissing
|
||||
expr: |
|
||||
(count(up == 0) / count(up)) > 0.10
|
||||
@@ -396,7 +396,7 @@ spec:
|
||||
annotations:
|
||||
summary: "More than 10% of Prometheus targets are down"
|
||||
description: "Over 10% of scrape targets have been down for 15m — a broad monitoring blind spot."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/targets-down"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/targets-down"
|
||||
|
||||
# =====================================================================================
|
||||
# Automated updates — Flux image-automation + hands-off update health (docs/11, ADR-0009)
|
||||
@@ -430,7 +430,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Flux image-update-automation failing: {{ $labels.name }}"
|
||||
description: "ImageUpdateAutomation {{ $labels.namespace }}/{{ $labels.name }} cannot reconcile (git write-back) for 30m — stable image advances have stopped (R-UPD-02). Check the git write credential."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/image-automation"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/image-automation"
|
||||
# An ImagePolicy/ImageRepository can't resolve the latest stable tag (registry rate-limit /
|
||||
# auth / bad range) → that service stops receiving updates even though automation is healthy.
|
||||
- alert: ImagePolicyStalled
|
||||
@@ -444,7 +444,7 @@ spec:
|
||||
annotations:
|
||||
summary: "Image scan/policy stalled: {{ $labels.kind }}/{{ $labels.name }}"
|
||||
description: "{{ $labels.kind }} {{ $labels.namespace }}/{{ $labels.name }} cannot resolve its stable tag for 1h (registry rate-limit / creds / range) — no longer receiving updates (R-UPD-01)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/image-policy-stalled"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/image-policy-stalled"
|
||||
# A bad update was health-gated and AUTO-ROLLED-BACK (or is stuck mid-remediation). helm-controller
|
||||
# holds Ready=False while a failed upgrade remediates/rolls back to the last-good revision
|
||||
# (HelmRelease.spec.upgrade.remediation + .spec.rollback). This is the R-UPD-03 safety rail firing.
|
||||
@@ -461,4 +461,4 @@ spec:
|
||||
annotations:
|
||||
summary: "Update health-gate failed / rolled back: {{ $labels.name }}"
|
||||
description: "HelmRelease {{ $labels.namespace }}/{{ $labels.name }} failed its health-gated upgrade and is rolling back to the last-good revision (bad-update auto-rollback — R-UPD-03). Consider an `update.pin` escape hatch (docs/11)."
|
||||
runbook_url: "https://home.keithjackson.dev/runbooks/update-rollback"
|
||||
runbook_url: "https://${BASE_DOMAIN}/runbooks/update-rollback"
|
||||
@@ -14,7 +14,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- grafana.home.keithjackson.dev
|
||||
- grafana.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -194,15 +194,15 @@ spec:
|
||||
optional: true
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.home.keithjackson.dev
|
||||
root_url: https://grafana.${BASE_DOMAIN}
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Keycloak
|
||||
client_id: grafana
|
||||
scopes: "openid email profile"
|
||||
auth_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/auth
|
||||
token_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/token
|
||||
api_url: https://keycloak.home.keithjackson.dev/realms/homelab/protocol/openid-connect/userinfo
|
||||
auth_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/auth
|
||||
token_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/token
|
||||
api_url: https://keycloak.${BASE_DOMAIN}/realms/homelab/protocol/openid-connect/userinfo
|
||||
login_attribute_path: preferred_username
|
||||
email_attribute_path: email
|
||||
# Map Keycloak groups (the `groups` claim) to Grafana roles (R-IDP-06): platform-admins ->
|
||||
|
||||
@@ -32,7 +32,7 @@ spec:
|
||||
existingSecret: oauth2-proxy-credentials
|
||||
extraArgs:
|
||||
provider: oidc
|
||||
oidc-issuer-url: https://keycloak.home.keithjackson.dev/realms/homelab
|
||||
oidc-issuer-url: https://keycloak.${BASE_DOMAIN}/realms/homelab
|
||||
# Forward-auth (ext-auth) mode: no upstream app; return 202 for an authenticated session so the
|
||||
# Gateway ext-authz can allow/deny. Auth headers are surfaced to the backend.
|
||||
reverse-proxy: "true"
|
||||
@@ -41,9 +41,9 @@ spec:
|
||||
pass-access-token: "true"
|
||||
email-domain: "*"
|
||||
# Single sign-in/callback host for all forward-auth-protected services on the platform zone.
|
||||
redirect-url: https://auth.home.keithjackson.dev/oauth2/callback
|
||||
whitelist-domain: ".home.keithjackson.dev"
|
||||
cookie-domain: ".home.keithjackson.dev"
|
||||
redirect-url: https://auth.${BASE_DOMAIN}/oauth2/callback
|
||||
whitelist-domain: ".${BASE_DOMAIN}"
|
||||
cookie-domain: ".${BASE_DOMAIN}"
|
||||
cookie-secure: "true"
|
||||
# Expose group membership so per-service group authorization (R-IDP-06) can be enforced. The
|
||||
# Keycloak `oauth2-proxy` client emits a `groups` claim (full.path=false => bare names like
|
||||
|
||||
@@ -14,7 +14,7 @@ spec:
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- auth.home.keithjackson.dev
|
||||
- auth.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "DNS label -> <name>.home.keithjackson.dev (docs/13, docs/09). Must be a valid RFC 1123 DNS label.",
|
||||
"description": "DNS label -> <name>.${BASE_DOMAIN}, i.e. <name>.home.keithjackson.dev on dev and <name>.prod.home.keithjackson.dev on prod (docs/13, docs/09). Must be a valid RFC 1123 DNS label.",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
|
||||
"minLength": 1,
|
||||
"maxLength": 63
|
||||
@@ -223,8 +223,8 @@
|
||||
"properties": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": "FQDN under the platform zone. Must be a <label>.home.keithjackson.dev name (docs/09).",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.home\\.keithjackson\\.dev$"
|
||||
"description": "Hostname under the platform zone, as a single DNS label plus the ${BASE_DOMAIN} token — e.g. 'gitea.${BASE_DOMAIN}' (docs/09). ${BASE_DOMAIN} is the per-environment hostname suffix from clusters/base/cluster-vars.yaml, resolved by Flux postBuild substitution at reconcile time (dev: home.keithjackson.dev, prod: prod.home.keithjackson.dev). AUTHOR THE TOKEN, NOT A LITERAL: a literal suffix is what let prod be authored to serve dev's public hostnames out of the shared Cloudflare zone, and prod's external-dns then deleted dev's records (infra/PROD-PROVISIONING.md §10). The literal 'home.keithjackson.dev' form stays accepted so the L1 render fixtures — which are renderer INPUTS, not reconciled manifests — keep validating unchanged.",
|
||||
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.(\\$\\{BASE_DOMAIN\\}|home\\.keithjackson\\.dev)$"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
|
||||
Reference in new issue
Block a user