Semaphore's whole SEMAPHORE_OIDC_PROVIDERS JSON blob lived inside the SOPS-encrypted
semaphore-bootstrap Secret (key `oidc-providers-json`) because it CONTAINS the confidential
client secret. That buried two environment-specific, NON-SECRET URLs in ciphertext —
provider_url (the realm issuer) and redirect_url — where Flux's ${BASE_DOMAIN} postBuild
substitution cannot reach. Both were frozen at `home.keithjackson.dev`, so prod's Semaphore
was pointed at dev's Keycloak and dev's own redirect URI: identical root cause to the
Headlamp login timeout, not yet reported only because nobody had tried prod SSO.
FIX: split the blob. The URLs are public and now live as a plaintext literal in
catalog/semaphore.yaml (rendered to the apps/wave-1 golden), ${BASE_DOMAIN}-substituted like
every other hostname. Only the client secret stays encrypted, in its own key
`oidc-client-secret`, and the KUBELET splices it back via dependent-env-var expansion:
$(SEMAPHORE_OIDC_CLIENT_SECRET) resolves against the secretKeyRef env var declared
immediately above it (order is load-bearing — the kubelet resolves env in list order and
leaves a forward reference as literal text).
Verified: the new key holds the same value as keycloak-realm-clients/semaphore-client-secret
(the mirror invariant), Semaphore v2.19.10's util.OidcProvider consumes client_secret as
before, and `flux envsubst --strict` leaves $(...) untouched while resolving ${BASE_DOMAIN} —
dev renders keycloak.home.keithjackson.dev, prod keycloak.prod.home.keithjackson.dev.
apps/wave-1/semaphore.yaml is a render golden; regenerated with the canonical
`catalog/render.py --all catalog -o apps --wave-layout --no-image-automation`.
Co-Authored-By: Claude Opus 5 <[email protected]>
288 lines
7.1 KiB
YAML
288 lines
7.1 KiB
YAML
---
|
|
# Namespace (trust zone; default-deny + policies apply)
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: semaphore
|
|
labels:
|
|
homelab.keithjackson.dev/app-namespace: 'true'
|
|
gateway-access: shared
|
|
annotations:
|
|
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
|
---
|
|
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: default-deny
|
|
namespace: semaphore
|
|
labels:
|
|
homelab.keithjackson.dev/baseline: default-deny
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes:
|
|
- Ingress
|
|
- Egress
|
|
ingress: []
|
|
egress:
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: kube-system
|
|
ports:
|
|
- protocol: UDP
|
|
port: 53
|
|
- protocol: TCP
|
|
port: 53
|
|
---
|
|
# PVC semaphore-config (R-STATE)
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: semaphore-config
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
---
|
|
# Deployment (R-PLAT-02)
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: semaphore
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: semaphore
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
homelab.keithjackson.dev/egress-class: internet
|
|
homelab.keithjackson.dev/auth-mode: oidc
|
|
annotations:
|
|
homelab.keithjackson.dev/auth-mode: oidc
|
|
homelab.keithjackson.dev/auth-groups: platform-admins
|
|
spec:
|
|
containers:
|
|
- name: semaphore
|
|
image: semaphoreui/semaphore:v2.19.10
|
|
ports:
|
|
- containerPort: 3000
|
|
env:
|
|
- name: SEMAPHORE_DB_DIALECT
|
|
value: postgres
|
|
- name: SEMAPHORE_DB_HOST
|
|
value: semaphore-db-rw.semaphore-db.svc.cluster.local
|
|
- name: SEMAPHORE_DB_PORT
|
|
value: '5432'
|
|
- name: SEMAPHORE_DB
|
|
value: semaphore
|
|
- name: SEMAPHORE_DB_USER
|
|
value: semaphore
|
|
- name: SEMAPHORE_DB_PASS
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: semaphore-bootstrap
|
|
key: db-password
|
|
- name: SEMAPHORE_ADMIN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: semaphore-bootstrap
|
|
key: admin-username
|
|
- name: SEMAPHORE_ADMIN_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: semaphore-bootstrap
|
|
key: admin-password
|
|
- name: SEMAPHORE_ADMIN_NAME
|
|
value: Semaphore Admin
|
|
- name: SEMAPHORE_ADMIN_EMAIL
|
|
value: admin@${BASE_DOMAIN}
|
|
- name: SEMAPHORE_ACCESS_KEY_ENCRYPTION
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: semaphore-bootstrap
|
|
key: access-key-encryption
|
|
- name: SEMAPHORE_PORT
|
|
value: :3000
|
|
- name: SEMAPHORE_OIDC_CLIENT_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: semaphore-bootstrap
|
|
key: oidc-client-secret
|
|
- name: SEMAPHORE_OIDC_PROVIDERS
|
|
value: '{"keycloak":{"display_name":"Keycloak","provider_url":"https://keycloak.${BASE_DOMAIN}/realms/homelab","client_id":"semaphore","client_secret":"$(SEMAPHORE_OIDC_CLIENT_SECRET)","redirect_url":"https://semaphore.${BASE_DOMAIN}/api/auth/oidc/keycloak/redirect","scopes":["openid","profile","email"],"username_claim":"preferred_username","email_claim":"email","name_claim":"name","order":1}}'
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/semaphore
|
|
securityContext:
|
|
runAsUser: 1001
|
|
runAsGroup: 0
|
|
runAsNonRoot: true
|
|
fsGroup: 1001
|
|
volumes:
|
|
- name: config
|
|
persistentVolumeClaim:
|
|
claimName: semaphore-config
|
|
---
|
|
# Service
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: semaphore
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: semaphore
|
|
ports:
|
|
- name: port-3000
|
|
port: 3000
|
|
targetPort: 3000
|
|
---
|
|
# CiliumNetworkPolicy egress allow (R-NET-02): semaphore-allow-semaphore-db-5432
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: semaphore-allow-semaphore-db-5432
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: semaphore
|
|
egress:
|
|
- toEndpoints:
|
|
- matchLabels:
|
|
app.kubernetes.io/name: semaphore-db
|
|
k8s:io.kubernetes.pod.namespace: semaphore-db
|
|
toPorts:
|
|
- ports:
|
|
- port: '5432'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): semaphore-allow-gateway-ingress
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: semaphore-allow-gateway-ingress
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: semaphore
|
|
ingress:
|
|
- fromEntities:
|
|
- ingress
|
|
toPorts:
|
|
- ports:
|
|
- port: '3000'
|
|
protocol: TCP
|
|
---
|
|
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): semaphore-allow-internet-egress
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: semaphore-allow-internet-egress
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: semaphore
|
|
egress:
|
|
- toEntities:
|
|
- world
|
|
- toEndpoints:
|
|
- matchLabels:
|
|
io.kubernetes.pod.namespace: keycloak
|
|
app: keycloak
|
|
---
|
|
# HTTPRoute semaphore-3000 (R-TLS/R-DNS)
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata:
|
|
name: semaphore-3000
|
|
namespace: semaphore
|
|
labels:
|
|
app.kubernetes.io/name: semaphore
|
|
annotations:
|
|
homelab.keithjackson.dev/auth-mode: oidc
|
|
homelab.keithjackson.dev/auth-groups: platform-admins
|
|
spec:
|
|
parentRefs:
|
|
- name: shared
|
|
namespace: gateway
|
|
sectionName: https
|
|
hostnames:
|
|
- semaphore.${BASE_DOMAIN}
|
|
rules:
|
|
- matches:
|
|
- path:
|
|
type: PathPrefix
|
|
value: /
|
|
backendRefs:
|
|
- name: semaphore
|
|
port: 3000
|
|
---
|
|
# ServiceMonitor (R-OBS-01)
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: ServiceMonitor
|
|
metadata:
|
|
name: semaphore
|
|
namespace: semaphore
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: semaphore
|
|
namespaceSelector:
|
|
matchNames:
|
|
- semaphore
|
|
endpoints:
|
|
- port: metrics
|
|
interval: 30s
|
|
---
|
|
# PrometheusRule SLI alerts (R-OBS-01)
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: semaphore-slis
|
|
namespace: semaphore
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
app.kubernetes.io/name: semaphore
|
|
spec:
|
|
groups:
|
|
- name: semaphore.slis
|
|
rules:
|
|
- alert: semaphore-down
|
|
expr: up{job="semaphore"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: semaphore
|
|
annotations:
|
|
summary: semaphore is down
|
|
description: 'semaphore has been unreachable for 5m (SLI: up).'
|