home-network-k3s-impl/catalog/semaphore.yaml

# ServiceDefinition: semaphore
# Source of truth: spec docs/13-service-catalog.md row 10; docs/15-identity-and-access.md; docs/04.
# internal Ansible UI | egress internet | config on Longhorn + external PostgreSQL | auth oidc.
#
# Semaphore (semaphoreui/semaphore) is a web UI for running Ansible/Terraform. It is a stateless
# app over (a) its /etc/semaphore config PVC on Longhorn and (b) an external PostgreSQL — the
# SECOND CNPG-DB app, reusing the pattern nextcloud established (apps/nextcloud-db.yaml). Its DB is
# a CloudNativePG Cluster in a dedicated namespace (apps/semaphore-db.yaml).
#
# DB WIRING (schema >=1.2.0 env): the app reaches its PostgreSQL cross-namespace at
# semaphore-db-rw.semaphore-db:5432 (the CNPG read-write Service). Its owner credentials are the
# SAME password held in two SOPS secrets — semaphore-db-app (basic-auth, ns semaphore-db; consumed
# by the CNPG Cluster's initdb) and semaphore-bootstrap (ns semaphore; consumed here as db-password).
# A Pod can only read Secrets in its OWN namespace, so the shared password is duplicated across the
# two encrypted files rather than replicated cross-namespace (documented in apps/semaphore-db.yaml).
#
# PERSISTENT ENCRYPTION KEY: SEMAPHORE_ACCESS_KEY_ENCRYPTION is a base64 32-byte AES key Semaphore
# uses to encrypt the SSH keys / access credentials it stores in the DB. It MUST be FIXED and stable
# across restarts — if it changes, every stored credential becomes undecryptable. It is pinned via
# the semaphore-bootstrap SOPS secret (like gitea's SECRET_KEY), never regenerated per boot.
# Validates against schemas/servicedefinition.schema.json.
name: semaphore
kind: container
source:
  image: semaphoreui/semaphore:v2.19.17   # pinned recent stable tag (R-UPD-01); update.range governs advancement
env:
  # --- PostgreSQL connection (the CNPG-DB pattern). DIALECT/HOST/PORT/DB/USER are literals; PASS is
  #     sourced from the in-namespace SOPS secret (the CNPG -app secret lives in ns semaphore-db,
  #     unreachable by a Pod secretKeyRef, so the same value is mirrored into semaphore-bootstrap). ---
  - name: SEMAPHORE_DB_DIALECT
    value: postgres
  - name: SEMAPHORE_DB_HOST
    value: semaphore-db-rw.semaphore-db.svc.cluster.local
  - name: SEMAPHORE_DB_PORT
    value: "5432"
  - name: SEMAPHORE_DB
    value: semaphore
  - name: SEMAPHORE_DB_USER
    value: semaphore
  - name: SEMAPHORE_DB_PASS
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
        key: db-password
  # --- Break-glass local admin (docs/15 availability caveat): created on first-boot so the operator
  #     is never locked out by an IdP outage. Username/password sourced from the SOPS secret; the
  #     display name + email are non-sensitive literals. ---
  - name: SEMAPHORE_ADMIN
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
        key: admin-username
  - name: SEMAPHORE_ADMIN_PASSWORD
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
        key: admin-password
  - name: SEMAPHORE_ADMIN_NAME
    value: Semaphore Admin
  - name: SEMAPHORE_ADMIN_EMAIL
    value: admin@${BASE_DOMAIN}
  # --- Persistent access-key encryption key (see header): FIXED base64 32-byte key from SOPS so the
  #     SSH keys / credentials Semaphore stores in the DB stay decryptable across restarts. ---
  - name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
        key: access-key-encryption
  # Because the Service is named `semaphore`, the kubelet injects a Docker-link-style
  # SEMAPHORE_PORT=tcp://<svcIP>:3000 env var, which COLLIDES with Semaphore's OWN SEMAPHORE_PORT
  # (its HTTP listen port) and crashes config validation ("field 'Port' is not valid: tcp://..."). An
  # explicit env entry shadows the injected one (the kubelet gives user-defined env precedence over
  # service links), pinning the listen port back to :3000 (matches containerPort/Service).
  - name: SEMAPHORE_PORT
    value: ":3000"
  # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). Semaphore
  #     v2.19.10 reads its OIDC providers from the SEMAPHORE_OIDC_PROVIDERS env as a JSON map
  #     (util ConfigType `oidc_providers`, env-tagged SEMAPHORE_OIDC_PROVIDERS). The value is a single
  #     JSON blob for the `keycloak` provider — provider_url is the homelab realm issuer (auto-discovery),
  #     client_id=semaphore, redirect_url=/api/auth/oidc/keycloak/redirect, claims map to the realm's
  #     token. The name is NOT a kubelet service-link var (those are SEMAPHORE_SERVICE_HOST /
  #     SEMAPHORE_PORT_*), so no collision. The LOCAL SEMAPHORE_ADMIN above stays break-glass (docs/15).
  #
  #     PER-ENVIRONMENT URLS (2026-09-07 fix). The blob USED to live whole inside the SOPS-encrypted
  #     semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential
  #     client secret. That hid two environment-specific, NON-SECRET URLs — the realm issuer and the
  #     redirect URI — inside ciphertext, where Flux's ${BASE_DOMAIN} postBuild substitution CANNOT
  #     reach them. Both were frozen at the apex `home.keithjackson.dev`, so PROD (BASE_DOMAIN =
  #     prod.home.keithjackson.dev) pointed its OIDC at DEV's Keycloak — unroutable from prod's
  #     192.168.2.0/24 (the identical root cause as the Headlamp login timeout; see
  #     platform/headlamp/README.md). Dev was only ever "right" because dev IS the apex.
  #
  #     THE SPLIT: the URLs are public, so the blob is now a PLAINTEXT literal here and gets
  #     ${BASE_DOMAIN}-substituted per environment like every other hostname. Only the confidential
  #     client secret stays SOPS-encrypted, in its own key (`oidc-client-secret`), and is spliced back
  #     in by the KUBELET's dependent-env-var expansion: an env `value` may reference an EARLIER env
  #     var of the same container as $(NAME), and secretKeyRef-sourced vars are eligible. So
  #     SEMAPHORE_OIDC_CLIENT_SECRET MUST STAY IMMEDIATELY ABOVE — the kubelet resolves env in list
  #     order, and a reference to a not-yet-defined name is left as the LITERAL text $(...). It is
  #     fail-loud in the way that matters: if the Secret key is missing the Pod never starts
  #     (CreateContainerConfigError), rather than silently authenticating against nothing. ---
  - name: SEMAPHORE_OIDC_CLIENT_SECRET
    valueFrom:
      secretKeyRef:
        name: semaphore-bootstrap
        key: oidc-client-secret
  - name: SEMAPHORE_OIDC_PROVIDERS
    value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
# The semaphoreui/semaphore image runs as the non-root user UID 1001 / GID 0 (tini -> server-wrapper),
# and on boot the wrapper writes its generated config.json into the config dir. A mounted Longhorn PVC
# is root:root, so UID 1001 cannot write it -> the pod panics ("permission denied"). fsGroup=1001 makes
# the kubelet group-own the mounted PVC to 1001 and add it as a supplemental group so the process can
# write; runAsUser/runAsGroup pin the image's own identity, runAsNonRoot asserts it (R-STATE-01/02).
podSecurityContext:
  runAsUser: 1001
  runAsGroup: 0
  runAsNonRoot: true
  fsGroup: 1001
exposure: internal              # LAN-only (internal DNS + Gateway on LAN); not edge-forwarded (docs/09; R-DNS/R-NET)
egress: internet                # normal WAN egress (docs/03; R-NET-05) — clones repos, installs Galaxy roles
storage:
  - name: config
    size: 1Gi
    class: longhorn             # config (generated config.json, tmp inventories) on the 3-replica tier
    mountPath: /etc/semaphore   # Semaphore reads/writes its config under /etc/semaphore (R-STATE-02)
dependencies:
  - to: semaphore-db            # explicit allow: Semaphore -> its PostgreSQL (docs/13 graph; R-NET-02).
    port: 5432                  # Renders the egress half; the DB namespace mirrors keycloak-db (no
    protocol: tcp               # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
ingressRoutes:
  - host: semaphore.${BASE_DOMAIN}
    port: 3000                  # Semaphore HTTP web/API; wildcard TLS terminates at the shared Gateway (docs/08/09)
auth:
  mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
  groups:
    - platform-admins           # access scoped to an authorized group (R-IDP-06)
  # WIRED (native OIDC): the Keycloak `semaphore` confidential client (platform/keycloak/keycloak-realm-
  # import.yaml + live kcadm) is consumed via the SEMAPHORE_OIDC_PROVIDERS env — a PLAINTEXT JSON blob
  # (see `env` above) whose per-environment URLs are ${BASE_DOMAIN}-substituted, with the confidential
  # client secret spliced in at runtime from the semaphore-bootstrap Secret key `oidc-client-secret`.
  # The `groups` claim is exposed for authorization (R-IDP-06). Semaphore KEEPS its LOCAL admin as the
  # break-glass path so an IdP outage never locks the operator out (docs/15 "Availability caveat").
  # No secret is embedded here.
update:
  channel: stable               # R-UPD-01
  range: ">=2.19.0 <3.0.0"      # stay on the 2.x stable line
  cadence: "0 4 * * *"
  autoRollbackOnFailedHealth: true   # R-UPD-03
observability:
  metrics: true
  slis:
    - up                        # automation UI down is a critical alert (docs/10; R-OBS-01)
backup:
  policy: daily-offcluster      # DR-critical: its DB holds automation credentials, backed up off-cluster (docs/13, docs/04; R-STATE-03)