--- # clusters/dev — the ACTIVE build target overlay. # # `flux bootstrap ... --path=clusters/dev` points the dev cluster's Flux here. It reuses the base # reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02). # It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git # on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1). apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base - sync.yaml # self-managing root Kustomization → ./clusters/dev patches: # --- SOPS decryption + bootstrap source (PM-inlined from bootstrap/flux/patches/, 2026-08-16) --- # Inlined (not cross-dir file refs) to stay within kustomize's default load restrictions. # Realises the commented decryption TODOs in clusters/base and sets the real source URL. # The `sops-age` Secret is created out-of-band (bootstrap/sops/install-age-key.sh) — name only. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: platform namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # SOPS decryption on EACH app wave (the app tier is now apps-wave-0..N, not a single `apps` # Kustomization). Same stanza the retired `apps` Kustomization received — one per wave, by # name. Add a patch here whenever `render.py --emit-waves` introduces a new deepest wave # (DESIGN-waves.md cutover note). The `sops-age` Secret is created out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-0 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-1 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-2 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # oauth2-proxy moved to its own post-platform Flux Kustomization (clusters/base/oauth2-proxy.yaml). # It carries oauth2-proxy-credentials.sops.yaml, previously decrypted via the `platform` # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: oauth2-proxy namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # headlamp moved to its own post-platform Flux Kustomization (clusters/base/headlamp.yaml). # It carries headlamp-oidc-credentials.sops.yaml, previously decrypted via the `platform` # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: headlamp namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # crafty has its own post-platform Flux Kustomization (clusters/base/crafty.yaml). It carries # crafty-oauth2-proxy-credentials.sops.yaml, so it needs its own SOPS decryption stanza (same as # headlamp/oauth2-proxy). The `sops-age` Secret is created out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: crafty namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml). # It carries BOTH halves of the per-env `cloud` INI credential — velero-cloud.sops.yaml (dev's scoped # MinIO key) and velero-cloud-prod.sops.yaml (prod's R2 token) — so it needs its own SOPS decryption # stanza (same as headlamp/oauth2-proxy/crafty). Which one the Deployment mounts is chosen by # BACKUP_CREDENTIALS_SUFFIX in the cluster-vars patch below, not here. The `sops-age` Secret is # out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: velero namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # vpn-ingress (the WireGuard remote-access VPN, clusters/base/vpn-ingress.yaml) carries # platform/vpn-ingress/wg-server-key.sops.yaml — the WireGuard SERVER private key, the one # secret in that component. Without this stanza kustomize hands Flux the raw ciphertext and the # Secret applies full of ENC[...]: the APPLY SUCCEEDS and every Flux condition stays green, and # the only symptom is a VPN that no client can handshake with. The DaemonSet's init container # validates the value with `wg pubkey` specifically so that state is a named CrashLoop instead. # The `sops-age` Secret is out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: vpn-ingress namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # cloudflared (the Cloudflare Tunnel connector, clusters/base/cloudflared.yaml) carries # platform/cloudflared/cloudflared-credentials.sops.yaml — the tunnel credentials JSON, the one # secret in that component. Without this stanza the Secret applies full of ENC[...]: the APPLY # SUCCEEDS, the Kustomization stays Ready, and the only symptom is a connector that cannot # authenticate to the edge (CloudflaredTunnelDown fires). The `sops-age` Secret is out-of-band # (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: cloudflared namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # pm-agent (the Program Manager agent service, clusters/base/pm-agent.yaml) carries # pm-agent-credentials.sops.yaml — the Gitea write token for the dedicated non-admin bot # account and the Anthropic subscription token from `claude setup-token`. Without this # stanza kustomize hands Flux the raw SOPS document and the Secret applies full of # ciphertext: the APPLY SUCCEEDS, and the only symptom is a dispatcher that cannot # authenticate to Gitea — so this stanza is as load-bearing as the file it decrypts (same # silent-failure shape as the apps-vms note below/above). The `sops-age` Secret is created # out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: pm-agent namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # apps-vms (the KubeVirt VM tier, clusters/base/apps-vms.yaml) gained its FIRST encrypted Secret # on 2026-09-14: apps/vms/unifi/unifi-uos-installer.sops.yaml, the operator's UOS installer URL, # which 30-virtualmachine.yaml mounts into the guest as a disk. Without this stanza kustomize # would hand Flux the raw SOPS document and the VM would mount a disk full of ciphertext — the # failure is silent from the cluster's point of view (the Secret applies fine), so the stanza is # as load-bearing as the file itself. The `sops-age` Secret is out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-vms namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository metadata: name: flux-system namespace: flux-system spec: # Phase-6 cutover done: source is now the in-cluster Gitea; bootstrap host-Gitea retained # stopped as recovery. PUBLIC repo => Flux needs no creds; Flux resolves the host in-cluster # via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO # secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and # revert this url (docs/05). url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git ref: branch: main # --- dev-scoped cluster-vars (R-ENV-02) --- - target: kind: ConfigMap name: cluster-vars patch: |- - op: replace path: /data/CLUSTER_ENV value: "dev" # DNS-01 via LE PRODUCTION: in-cluster OIDC clients (oauth2-proxy, Grafana) must trust the # wildcard cert for SSO to work; the staging root isn't in any trust store. Prod issuer is # registered + Ready; one wildcard cert is well within LE rate limits — ADR-0006, R-TLS-01/03. - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" # --- DNS identity (R-DNS-01/02/03) ----------------------------------------------------- # DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly # like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running. # Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its # Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev # (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to # prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did # not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration, # not a variable change: infra/PROD-PROVISIONING.md §10. - op: replace path: /data/BASE_DOMAIN value: "home.keithjackson.dev" # ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself, # and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped # the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held. - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER value: "keithjackson.dev" # DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT # registry under owner `default` (the external-dns chart default, which is what both clusters # silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records # BY this string: any other value and it stops recognising every record it manages, then # re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the # whole point. The base default is a non-colliding sentinel, so this line is load-bearing. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID value: "default" # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) ----------------------------- # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it # equals the base default. These values are what the LIVE dev cluster is running right now, so # the substituted output is byte-identical to the pre-parameterization hardcoded manifests # (verified 2026-08-31). Never let this block drift from the live cluster. # # Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN # (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers # *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. - op: replace path: /data/GATEWAY_VIP value: "192.168.1.53" # k3s API endpoint Cilium's kube-proxy replacement dials — DEV'S CONTROL-PLANE VIP, # the address kube-vip floats, NOT the primary node .50. # [parity] equals the Ansible inventory's k3s_apiserver_endpoint for dev # (= kube_vip_address, group_vars/all.yml). See clusters/base/cluster-vars.yaml for why. # # *** THE ONE EXCEPTION TO "NEVER LET THIS BLOCK DRIFT FROM THE LIVE CLUSTER" *** # Every other value in this block is what dev is running right now. This one is not: # the live dev cilium-agents dial 192.168.1.50 (k3s-1). Changing it here is a # DELIBERATE, OPERATOR-APPROVED move (2026-09-06) and reconciling it ROLLS THE CILIUM # AGENT DAEMONSET on dev — one node at a time (updateStrategy.maxUnavailable: 1, kept # in parity for exactly this reason). Do not push this to dev-live casually, and not # while dev's DNS migration is mid-flight; the L2 datapath survives an agent restart # but a wrong value here is the 2026-09-01 incident. .49 is in dev's serving-cert SANs # already (k3s_tls_sans) and dev's own kubeconfig validates against it via # tls-server-name, so TLS is not the risk — the risk is reachability, and .49 is on the # same LAN as the nodes and ARP-announced by kube-vip on every server. - op: replace path: /data/K3S_API_HOST value: "192.168.1.49" # Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs # (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free. - op: replace path: /data/CILIUM_LB_POOL_START value: "192.168.1.53" - op: replace path: /data/CILIUM_LB_POOL_STOP value: "192.168.1.60" # Pinned L4 VIPs out of that pool (lbipam.cilium.io/ips). Changing either MOVES a live device- # facing endpoint: unifi is the inform/STUN target every adopted UniFi device holds. - op: replace path: /data/UNIFI_L4_VIP value: "192.168.1.54" - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.1.55" # Bulk (external NFS) storage host. THIS VALUE IS INERT — dev is decommissioned, its nodes # (192.168.1.40/.41/.42) no longer exist, and the Proxmox host this used to name (192.168.1.9) # went with the pools it held. It is restated per this overlay's "supply every knob" # convention, not because anything consumes it. It carries the real address so that no file # in the tree renders a host that is gone; it has NOT been proven from a dev node, because # there is no dev node to prove it from. - op: replace path: /data/BULK_NFS_SERVER value: "192.168.2.30" # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG # barmanObjectStore and Velero's backupStorageLocation write to. # # DEV IS DELIBERATELY STAYING ON THIS MinIO while prod goes to Cloudflare R2 (2026-09-07). # Prod could not use it at all — it cannot even route to it — but dev's move is a separate, # operator-gated decision, because R2 would start dev on an EMPTY prefix: Velero would write # a fresh kopia repository, CNPG would take new base backups, and the 30-day Barman window # would rebuild from zero. Dev's DR restore is the one PROVEN recovery path in this repo # (R-STATE-03); resetting its PITR window is not a side effect to take while fixing prod. - op: replace path: /data/S3_BACKUP_ENDPOINT value: "http://192.168.1.9:9000" # DO NOT CHANGE THESE FOUR. They are dev's LIVE object layout, restated verbatim for the same # reason as LONGHORN_DATA_PATH and EXTERNALDNS_TXT_OWNER_ID below/above: they equal the base # defaults today, and a silent drift in base would ORPHAN dev's existing backups — the ones # its proven restore depends on — without any manifest looking wrong. # # `us-east-1` is what dev's Velero has signed every request with since bootstrap; MinIO # ignores the value, but changing it is a needless risk against a live, working backup path. # An EMPTY VELERO_BACKUP_PREFIX means the bucket root, which is where dev's backups already # are — and the Velero chart omits `objectStorage.prefix` entirely when it is empty (verified # against chart 12.1.0), so dev's live BackupStorageLocation is byte-unchanged by this. - op: replace path: /data/S3_BACKUP_REGION value: "us-east-1" - op: replace path: /data/VELERO_BACKUP_BUCKET value: "velero" - op: replace path: /data/VELERO_BACKUP_PREFIX value: "" - op: replace path: /data/CNPG_BACKUP_PATH value: "s3://cnpg" # WHICH CREDENTIAL SIGNS THOSE REQUESTS. Empty suffix = the UNSUFFIXED Secrets # (velero-cloud-credentials, -backup-credentials) — the scoped MinIO keys dev's Velero and # its three CNPG Clusters are running RIGHT NOW, byte-identical to what is on the cluster. Prod # renders "-prod" and reads the Cloudflare R2 token from the parallel `-prod` Secrets instead. # # RESTATED EXPLICITLY even though it equals the base default, for the same reason as # EXTERNALDNS_TXT_OWNER_ID and the four backup-layout knobs above: the four Secrets are SHARED # base manifests, so before this split prod's R2 token landed in the objects DEV reads. Dev # signing MinIO with an R2 key id is 403 on every barman-cloud-wal-archive, and archive_mode=on # turns that into a filling pgdata PVC — the disk-full clock, on the one cluster whose DR # restore is PROVEN (R-STATE-03). This line is what keeps dev on its own credential. # # DEV IS NOT BEING BACKED UP GOING FORWARD and is slated for decommission (operator, # 2026-09-07), so this is not protecting a future backup — it is protecting the EXISTING # backups and the WAL chain until dev is retired. Do not "simplify" it away while dev runs. - op: replace path: /data/BACKUP_CREDENTIALS_SUFFIX value: "" # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas. - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT value: "3" # DO NOT CHANGE THIS VALUE. The LIVE dev cluster has 18 Longhorn volumes whose replicas are # on disk under /var/lib/longhorn — the dedicated /dev/vdb ext4 disk roles/node_baseline # mounts there. Longhorn addresses a replica by dataPath; pointing defaultDataPath anywhere # else would orphan every one of them (they are not recoverable from the old path by # rescheduling). Restated explicitly even though it equals the base default, exactly like the # LAN block above: this is what the live cluster is running. Prod uses /data/longhorn because # its data partition also carries k3s's data dir — see clusters/prod and PROD-PROVISIONING §2. - op: replace path: /data/LONGHORN_DATA_PATH value: "/var/lib/longhorn" # smaller retention / volumes to keep the dev workstation footprint light. - op: replace path: /data/MONITORING_PROMETHEUS_RETENTION value: "3d" # 5Gi was too tight: 3d of this cluster's series volume (~1.2Gi/day of blocks) plus a WAL that # ballooned past 1Gi overran the disk, wedging WAL commits and firing every absent()/==0 alert # falsely. Bumped to 8Gi (still lighter than base 10Gi) so the size-cap below has real headroom. - op: replace path: /data/MONITORING_PROMETHEUS_PVC_SIZE value: "8Gi" # Size-cap kept well below the 8Gi PVC (headroom for WAL + head + compaction scratch) so # Prometheus prunes oldest blocks by size and never fills the disk on this young cluster. - op: replace path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE value: "5GiB" - op: replace path: /data/MONITORING_LOKI_PVC_SIZE value: "5Gi" # single-replica control-plane addons in dev (no HA needed to iterate). - op: replace path: /data/KEYCLOAK_REPLICAS value: "1" - op: replace path: /data/OAUTH2_PROXY_REPLICAS value: "1" - op: replace path: /data/VELERO_RETENTION value: "72h" # dev-softened Kyverno for cold-start (base defaults Enforce/Fail/3 = prod posture). # Not credential-driven — bootstrap-safety knobs so a cold cluster comes up before # admission is proven/HA; flip dev to the strict posture once green. - op: replace path: /data/KYVERNO_VALIDATION_ACTION value: "Audit" - op: replace path: /data/KYVERNO_FORCE_FAILURE_POLICY_IGNORE value: "true" # 2 (not base-default 1): the webhook is fail-closed, so a single-pod restart 502s every # admission-gated write cluster-wide. Two replicas give a fail-closed webhook availability # headroom during a restart/upgrade while staying dev-modest (prod runs 3). - op: replace path: /data/KYVERNO_ADMISSION_REPLICAS value: "2"