diff --git a/apps/wave-1/semaphore-bootstrap-secret.sops.yaml b/apps/wave-1/semaphore-bootstrap-secret.sops.yaml index 490de04..20124f5 100644 --- a/apps/wave-1/semaphore-bootstrap-secret.sops.yaml +++ b/apps/wave-1/semaphore-bootstrap-secret.sops.yaml @@ -5,6 +5,15 @@ # credential from its own namespace (a Pod cannot read a Secret in another namespace). # access-key-encryption: FIXED base64 32-byte AES key Semaphore uses to encrypt stored SSH keys / # credentials; MUST stay stable across restarts or stored secrets become undecryptable. +# oidc-client-secret: the Keycloak `semaphore` CONFIDENTIAL client secret, and ONLY that. It MIRRORS +# `semaphore-client-secret` in platform/keycloak/keycloak-realm-clients.sops.yaml (the realm import +# injects that copy into the client), same cross-namespace mirror pattern as db-password above. +# It replaced the former `oidc-providers-json` key (2026-09-07): that key held the WHOLE +# SEMAPHORE_OIDC_PROVIDERS blob, which buried the environment-specific realm-issuer and redirect +# URLs inside ciphertext where Flux's ${BASE_DOMAIN} postBuild substitution could not reach them, +# so prod ran with dev's URLs. Those URLs are PUBLIC and now live in plaintext in +# catalog/semaphore.yaml (rendered to apps/wave-1/semaphore.yaml); the kubelet splices THIS value +# back into the blob via $(SEMAPHORE_OIDC_CLIENT_SECRET). Only the secret half is encrypted. # Recover: infra/devcontainer/dev.sh run sops -d apps/semaphore-bootstrap-secret.sops.yaml apiVersion: v1 kind: Secret @@ -15,11 +24,11 @@ metadata: app.kubernetes.io/name: semaphore type: Opaque stringData: - admin-username: ENC[AES256_GCM,data:uD/vO9A9MjerFg==,iv:i/NAqVD6rWg9oWWSGBrRkxzwcPp2IW1HLIKY0pdcPPg=,tag:r27tKLAJ+ydE7Ty3ccMNUw==,type:str] - admin-password: ENC[AES256_GCM,data:544nWY1UmBnlsdPJAQZJMXRumu4W9X7UXZvZ6vHN/ArjRgLXKTArOe5JkT/nNrNw,iv:AC1DDbZj2HfpU19EjzA7T+WpIaR4nZc3PNN61y8bAzU=,tag:r28LFi5w6ZeCCaRaX2jRdg==,type:str] - db-password: ENC[AES256_GCM,data:ySkdvaGRCW9fMQqChs5l8dY4zCG19i4HOEinRC48mgLuh9WOrjd16T6jwFgiwB55,iv:+1gztJ22HbwgQJiolZMMWOcKIsxrBJpnruJdtxdooOU=,tag:8CfJGwyPTjDGyr0CvalXMQ==,type:str] - access-key-encryption: ENC[AES256_GCM,data:1YAMLo1nVkpJWZc+KBZ7B4ia+Y8P+xMyOhHMcEGGJOXd3o+WC5FqirLBVh4=,iv:j28ozeaDhGWDjoaxFOhn2t9YbYE7L3ah/Mfi+i2DWwY=,tag:czG9Wxc/7Nsuh1W5iATOww==,type:str] - oidc-providers-json: ENC[AES256_GCM,data:90yt5mIVWLVI2KrlJWpxhL/BMDpEgRhv7+BL+mBRt79diRvU7pN9vtzhZprAKzcU1sEywlT4EwFlsvJpYyo5Qbkp+ZVknbyhAlGvHhJOBHukOPWgf4LignLerO2GzhXjOOzFqWfZ4EyUxNMWTbSe/xc0zIVHDDNQiYplGVRK+E/ukiI7sD/Tn2RrbrdvpC5GyK0NXWRsXn9ffhE3fCoiRHXagj9i/q+uUrQI5EKpPobrDUAGqvKoBXTl/wJ4phnWg9IqtCTE/c3yA7fLLz4j5dCtPZNSVPT4TKDx8Mdv8CfCgziVicjG2s+yHuWpsQn8lf6nvdJmpYPvIqTtLWdxFWuD8Vfo6Qbq2mqPbKR+KmDOGUnjl4gbZdL1SkE9JiqBUobz7UXeLXwYzxMIsinOfYZLcCk7alrD5Yndj9JDxDJZQ54sMMBfVbxDl94W1VHy03RxAR46KKAbPaxOP5wH4Q4z3SDceNwmuGCDRGChRXiHhN5X2X0GqYCbc8E6GdkIMcpmD80e26ydEmAjw7F89mCF4F/yMzJNiG82qaZIn8HeqgGA8urous9r9bYJNlitYTYQ,iv:jbfVdrrZ9fGiOxhQeoTJfFg1QwiOMkoQyaadiVfxX/w=,tag:JkepcRs34EuEgNb4buTTDQ==,type:str] + admin-username: ENC[AES256_GCM,data:ju3wVdjNzRUP/w==,iv:d/5ZxgQZ5bympu3gT2ck3DJy9iUPPBAtZYsJvD3To7I=,tag:0UtBaDVqNSK4RCJrQicchw==,type:str] + admin-password: ENC[AES256_GCM,data:pxIwpgh3BoND2TI0mPTixOTJoictzT77dqtuRdMKgY/2WaW4wM1Ra6H6n/dxnyR2,iv:eYbFQGFYVJsnYGwn6cOI3BgAdzMcin9YggAhiuL+xqM=,tag:/sPWGv4gue0vjc2nMk7wWA==,type:str] + db-password: ENC[AES256_GCM,data:hBFpWXdpk1R5a23Pntup03t56NBRPS+SZU8CrHVb/ypIaIpgf7kjYtVy6nhTF6u0,iv:eBzoQgH5ZhWWpfKZ6AQmwaSal5QaiuN9gfexQ7GsVFs=,tag:j2k8EwhKDlj/Ud6O39ioVQ==,type:str] + access-key-encryption: ENC[AES256_GCM,data:SKtAV4HxJ9pNYDqUK/1uOdI1rtMji3pTfprI2rhQxFQyhc6dFlhqI+OUEcY=,iv:rKCP+FLidcOltOwmMMcO5M8tz4qiifIMI+W5soGO2M8=,tag:HaHzkX/JI+9wmcg2SD9GAQ==,type:str] + oidc-client-secret: ENC[AES256_GCM,data:uCo7TNzbM2QKw9ie+xzyUG9MFKco1OAlwr5qPn/AGpNvo1YyXSfbZyyM0CLJ1NS821hGg5Zr/m3R/xpy84s3FQ==,iv:O8a4haskufpmorPaxOkJ1Q99Y5GTlzmDFJAsHS+irLk=,tag:VojJpA9jNMbNQmIe1Qr44w==,type:str] sops: kms: [] gcp_kms: [] @@ -29,14 +38,14 @@ sops: - recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuRDlqRFJ0dk14UHluSGRP - S1pESy9PWlpMakU0aTkzNVhqUGdtbWNEdTJjCm5RNG14azJHc28rdEJ4L0RzR0xO - MGRocGZ1V1JZTENWQkN3VGZzamp2eTgKLS0tIGt6QkNNaW5hdTBJVHY0SVo5d3JX - elp0SWxHcjg0RWF3aW9mWTJHZ0lhcUEKc0rU5VilAVDleCwqq4SOF+Y9sniu1Pka - w/MGDA9THSB0b+YgWRWiX/KiimFqp0ZFODOJ9cHVwFGY71+VadPwOg== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTzF6NTBtcFVWK1Vtc0I2 + VGdySHBsRGgvc1h0VzgvQ01xYVBJV3g2Q1NRCjJyZHVYZGtuamo1MFJHWlBwWkxt + cHA4VkY1TkRpN3M4UTdMOWZTbFVGemMKLS0tIGVNdDBrWVJveU56VjVBU2JWdHF4 + RGtaYStXYUpONTZjZCtmM0NyZ3Rka1UKIQpmQai8dz+uRAsXxyeuGaRlRwsFqIPQ + 9AfrZJcslYwlaK3ptm1oqaCdrlxcMar2nI7thSqzbLe+e1nzA2z7vQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-27T22:12:32Z" - mac: ENC[AES256_GCM,data:nHftJTGnTr0O5h8yGXwgz3eflggXJfIXM3m8n3bzDJE/P4nvSpZzFeMb+sc2on+AL+t+SeKrIl8blzm33XsQ1lkaY85Zm+bc5cnED1g04BMDVmmK3brh/8w50Bp2wPEVzb4I01MDzhhyIbOYg7pFvZAH3/VuqvM1t6XmfF1NrOA=,iv:4KBa9PH/dfdGwLC16RenZIVB6BtSoGo6x3qt3h1R43I=,tag:KULiA1klP/RZMNigkloGDA==,type:str] + lastmodified: "2026-09-07T07:01:50Z" + mac: ENC[AES256_GCM,data:Xutm1rQt57J4RILElD8FkNOsQT1HK5jZlPtqqLyZ/VqOoDwo+IS1wp7hROprlUvQC0bUXy79tRRIl5Q0CCOD+fvAGIdmade+vB7qXlYN6Q1441dGhuSTPvckso6FaB2ddRjKhKGLc/Z4bJoz9VaRNNxJNpWXIrENTg+I+QhNhlo=,iv:klJOrEQ6QN5dTGkQfGnbMbgCWsgWwZPb5YXDMeWeE34=,tag:fuJr4t3Tt3dT9u3/0SFo8Q==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.9.4 diff --git a/apps/wave-1/semaphore.yaml b/apps/wave-1/semaphore.yaml index d29a2e6..010e334 100644 --- a/apps/wave-1/semaphore.yaml +++ b/apps/wave-1/semaphore.yaml @@ -118,11 +118,13 @@ spec: key: access-key-encryption - name: SEMAPHORE_PORT value: :3000 - - name: SEMAPHORE_OIDC_PROVIDERS + - name: SEMAPHORE_OIDC_CLIENT_SECRET valueFrom: secretKeyRef: name: semaphore-bootstrap - key: oidc-providers-json + key: oidc-client-secret + - name: SEMAPHORE_OIDC_PROVIDERS + value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}' volumeMounts: - name: config mountPath: /etc/semaphore diff --git a/catalog/semaphore.yaml b/catalog/semaphore.yaml index 34ae9b4..a20855b 100644 --- a/catalog/semaphore.yaml +++ b/catalog/semaphore.yaml @@ -78,16 +78,34 @@ env: # (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single # JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery), # client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's - # token. It CONTAINS the confidential client secret, so the whole blob is held SOPS-encrypted in the - # semaphore-bootstrap Secret (key oidc-providers-json; same secret value SOPS-held in - # keycloak-realm-clients) and injected here by reference — never in cleartext. The name is NOT a - # kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / SEMAPHORE_PORT_*), so no collision. - # The LOCAL SEMAPHORE_ADMIN above stays as the break-glass path (docs/15). --- - - name: SEMAPHORE_OIDC_PROVIDERS + # token. The name is NOT a kubelet service-link var (those are SEMAPHORE_SERVICE_HOST / + # SEMAPHORE_PORT_*), so no collision. The LOCAL SEMAPHORE_ADMIN above stays break-glass (docs/15). + # + # PER-ENVIRONMENT URLS (2026-09-07 fix). The blob USED to live whole inside the SOPS-encrypted + # semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential + # client secret. That hid two environment-specific, NON-SECRET URLs — the realm issuer and the + # redirect URI — inside ciphertext, where Flux's ${BASE_DOMAIN} postBuild substitution CANNOT + # reach them. Both were frozen at the apex `home.keithjackson.dev`, so PROD (BASE_DOMAIN = + # prod.home.keithjackson.dev) pointed its OIDC at DEV's Keycloak — unroutable from prod's + # 192.168.2.0/24 (the identical root cause as the Headlamp login timeout; see + # platform/headlamp/README.md). Dev was only ever "right" because dev IS the apex. + # + # THE SPLIT: the URLs are public, so the blob is now a PLAINTEXT literal here and gets + # ${BASE_DOMAIN}-substituted per environment like every other hostname. Only the confidential + # client secret stays SOPS-encrypted, in its own key (`oidc-client-secret`), and is spliced back + # in by the KUBELET's dependent-env-var expansion: an env `value` may reference an EARLIER env + # var of the same container as $(NAME), and secretKeyRef-sourced vars are eligible. So + # SEMAPHORE_OIDC_CLIENT_SECRET MUST STAY IMMEDIATELY ABOVE — the kubelet resolves env in list + # order, and a reference to a not-yet-defined name is left as the LITERAL text $(...). It is + # fail-loud in the way that matters: if the Secret key is missing the Pod never starts + # (CreateContainerConfigError), rather than silently authenticating against nothing. --- + - name: SEMAPHORE_OIDC_CLIENT_SECRET valueFrom: secretKeyRef: name: semaphore-bootstrap - key: oidc-providers-json + key: oidc-client-secret + - name: SEMAPHORE_OIDC_PROVIDERS + value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}' # The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper), # and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC # is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes @@ -117,10 +135,12 @@ auth: groups: - platform-admins # access scoped to an authorized group (R-IDP-06) # WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm- - # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env (JSON blob held encrypted - # in the semaphore-bootstrap Secret, oidc-providers-json). The `groups` claim is exposed for - # authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the break-glass path so an IdP outage - # never locks the operator out (docs/15 "Availability caveat"). No secret is embedded here. + # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env — a PLAINTEXT JSON blob + # (see `env` above) whose per-environment URLs are ${BASE_DOMAIN}-substituted, with the confidential + # client secret spliced in at runtime from the semaphore-bootstrap Secret key `oidc-client-secret`. + # The `groups` claim is exposed for authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the + # break-glass path so an IdP outage never locks the operator out (docs/15 "Availability caveat"). + # No secret is embedded here. update: channel: stable # R-UPD-01 range: ">=2.19.0 <3.0.0" # stay on the 2.x stable line