--- # clusters/dev — the ACTIVE build target overlay. # # `flux bootstrap ... --path=clusters/dev` points the dev cluster's Flux here. It reuses the base # reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02). # It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git # on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1). # # ############################################################################################## # ### *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) *** ### # ### ### # ### This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex ### # ### `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url ### # ### moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a ### # ### merge to main IS the cutover — it executes on the live dev cluster with nobody present. ### # ### Dev's Gitea is both its GitOps source and the push target: if the new host is not up ### # ### with a Ready cert first, dev stops reconciling and pushing cannot repair it. ### # ### ### # ### Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator ### # ### present. Merging is step D3, not step one. ### # ############################################################################################## apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../base - sync.yaml # self-managing root Kustomization → ./clusters/dev patches: # --- SOPS decryption + bootstrap source (PM-inlined from bootstrap/flux/patches/, 2026-08-16) --- # Inlined (not cross-dir file refs) to stay within kustomize's default load restrictions. # Realises the commented decryption TODOs in clusters/base and sets the real source URL. # The `sops-age` Secret is created out-of-band (bootstrap/sops/install-age-key.sh) — name only. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: platform namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # SOPS decryption on EACH app wave (the app tier is now apps-wave-0..N, not a single `apps` # Kustomization). Same stanza the retired `apps` Kustomization received — one per wave, by # name. Add a patch here whenever `render.py --emit-waves` introduces a new deepest wave # (DESIGN-waves.md cutover note). The `sops-age` Secret is created out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-0 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-1 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: apps-wave-2 namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # oauth2-proxy moved to its own post-platform Flux Kustomization (clusters/base/oauth2-proxy.yaml). # It carries oauth2-proxy-credentials.sops.yaml, previously decrypted via the `platform` # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: oauth2-proxy namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # headlamp moved to its own post-platform Flux Kustomization (clusters/base/headlamp.yaml). # It carries headlamp-oidc-credentials.sops.yaml, previously decrypted via the `platform` # Kustomization; now that this tier owns the encrypted Secret it needs its own decryption stanza. - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: headlamp namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # crafty has its own post-platform Flux Kustomization (clusters/base/crafty.yaml). It carries # crafty-oauth2-proxy-credentials.sops.yaml, so it needs its own SOPS decryption stanza (same as # headlamp/oauth2-proxy). The `sops-age` Secret is created out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: crafty namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age # velero (backup controller) has its own post-platform Flux Kustomization (clusters/base/backup.yaml). # It carries velero-cloud.sops.yaml (the scoped MinIO `cloud` credential), so it needs its own SOPS # decryption stanza (same as headlamp/oauth2-proxy/crafty). The `sops-age` Secret is out-of-band (name only). - patch: |- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: name: velero namespace: flux-system spec: decryption: provider: sops secretRef: name: sops-age - patch: |- apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository metadata: name: flux-system namespace: flux-system spec: # Phase-6 cutover done: source is now the in-cluster Gitea; bootstrap host-Gitea retained # stopped as recovery. PUBLIC repo => Flux needs no creds; Flux resolves the host in-cluster # via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO # secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and # revert this url (docs/05). # # *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. *** # It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized — # it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO # postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and # break the source outright (the L0 flux-postbuild-substitution check asserts exactly this). # So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below. # # It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That # makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization # (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE # DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already # resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED # BY PUSHING — the broken thing is the path Flux uses to read pushes. # ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root # Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for # the migration window, and only restore the in-cluster url after the new host is proven. url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git ref: branch: main # --- dev-scoped cluster-vars (R-ENV-02) --- - target: kind: ConfigMap name: cluster-vars patch: |- - op: replace path: /data/CLUSTER_ENV value: "dev" # DNS-01 via LE PRODUCTION: in-cluster OIDC clients (oauth2-proxy, Grafana) must trust the # wildcard cert for SSO to work; the staging root isn't in any trust store. Prod issuer is # registered + Ready; one wildcard cert is well within LE rate limits — ADR-0006, R-TLS-01/03. - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" # ========================================================================================== # DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.** # ========================================================================================== # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day: # # dev -> dev.home.keithjackson.dev (this knob; = the base default, restated) # prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label, # and therefore no apex CNAME layer at all) # # *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right # now. Merging this file does not perform the migration — it DEFINES the target. The cutover is # an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10. # # WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20 # lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps # source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute, # DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does # not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED # BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery # is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair # in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host # FIRST, or park the source on the bootstrap Gitea for the window. # # ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh # ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s # server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's # split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex. - op: replace path: /data/BASE_DOMAIN value: "dev.home.keithjackson.dev" # ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING # F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a # strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching # record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it # newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held. - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER value: "keithjackson.dev" # OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex # records were already in Cloudflare's TXT registry under owner `default`, so renaming would # have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix # dev publishes a brand-new record set with no prior registry entries to recognise, so the id # is chosen on its merits, and `default` has none — it is the value external-dns uses when you # FORGET the flag, which is exactly how both clusters came to share it. Consequences, both # deliberate: # - dev's OLD apex records remain owned by `default`, which no environment now claims, so # nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first # publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing. # - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev # cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER. # dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID value: "dev" # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) ----------------------------- # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it # equals the base default. These values are what the LIVE dev cluster is running right now, so # the substituted output is byte-identical to the pre-parameterization hardcoded manifests # (verified 2026-08-31). Never let this block drift from the live cluster. # # Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN # (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers # *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. - op: replace path: /data/GATEWAY_VIP value: "192.168.1.53" # k3s API endpoint Cilium's kube-proxy replacement dials — the primary control-plane node # (k3s-1). [parity] equals the Ansible inventory's k3s_api_host for dev. - op: replace path: /data/K3S_API_HOST value: "192.168.1.50" # Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs # (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free. - op: replace path: /data/CILIUM_LB_POOL_START value: "192.168.1.53" - op: replace path: /data/CILIUM_LB_POOL_STOP value: "192.168.1.60" # Pinned L4 VIPs out of that pool (lbipam.cilium.io/ips). Changing either MOVES a live device- # facing endpoint: unifi is the inform/STUN target every adopted UniFi device holds. - op: replace path: /data/UNIFI_L4_VIP value: "192.168.1.54" - op: replace path: /data/CRAFTY_L4_VIP value: "192.168.1.55" # Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG # barmanObjectStore and Velero's backupStorageLocation write to. - op: replace path: /data/S3_BACKUP_ENDPOINT value: "http://192.168.1.9:9000" # dev mirrors prod's 3-node topology, so the replicated tier keeps 3 replicas. - op: replace path: /data/LONGHORN_DEFAULT_REPLICA_COUNT value: "3" # DO NOT CHANGE THIS VALUE. The LIVE dev cluster has 18 Longhorn volumes whose replicas are # on disk under /var/lib/longhorn — the dedicated /dev/vdb ext4 disk roles/node_baseline # mounts there. Longhorn addresses a replica by dataPath; pointing defaultDataPath anywhere # else would orphan every one of them (they are not recoverable from the old path by # rescheduling). Restated explicitly even though it equals the base default, exactly like the # LAN block above: this is what the live cluster is running. Prod uses /data/longhorn because # its data partition also carries k3s's data dir — see clusters/prod and PROD-PROVISIONING §2. - op: replace path: /data/LONGHORN_DATA_PATH value: "/var/lib/longhorn" # smaller retention / volumes to keep the dev workstation footprint light. - op: replace path: /data/MONITORING_PROMETHEUS_RETENTION value: "3d" # 5Gi was too tight: 3d of this cluster's series volume (~1.2Gi/day of blocks) plus a WAL that # ballooned past 1Gi overran the disk, wedging WAL commits and firing every absent()/==0 alert # falsely. Bumped to 8Gi (still lighter than base 10Gi) so the size-cap below has real headroom. - op: replace path: /data/MONITORING_PROMETHEUS_PVC_SIZE value: "8Gi" # Size-cap kept well below the 8Gi PVC (headroom for WAL + head + compaction scratch) so # Prometheus prunes oldest blocks by size and never fills the disk on this young cluster. - op: replace path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE value: "5GiB" - op: replace path: /data/MONITORING_LOKI_PVC_SIZE value: "5Gi" # single-replica control-plane addons in dev (no HA needed to iterate). - op: replace path: /data/KEYCLOAK_REPLICAS value: "1" - op: replace path: /data/OAUTH2_PROXY_REPLICAS value: "1" - op: replace path: /data/VELERO_RETENTION value: "72h" # dev-softened Kyverno for cold-start (base defaults Enforce/Fail/3 = prod posture). # Not credential-driven — bootstrap-safety knobs so a cold cluster comes up before # admission is proven/HA; flip dev to the strict posture once green. - op: replace path: /data/KYVERNO_VALIDATION_ACTION value: "Audit" - op: replace path: /data/KYVERNO_FORCE_FAILURE_POLICY_IGNORE value: "true" # 2 (not base-default 1): the webhook is fail-closed, so a single-pod restart 502s every # admission-gated write cluster-wide. Two replicas give a fail-closed webhook availability # headroom during a restart/upgrade while staying dev-modest (prod runs 3). - op: replace path: /data/KYVERNO_ADMISSION_REPLICAS value: "2"