diff --git a/catalog/render.py b/catalog/render.py index 16b4323..ef2c611 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -47,7 +47,8 @@ LBL = "homelab.keithjackson.dev" # The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal. # Every rendered manifest lands under a Flux path whose Kustomization declares # postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at -# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev. +# reconcile time: dev renders dev.home.keithjackson.dev, prod renders home.keithjackson.dev (the +# APEX — operator decision 2026-09-04; prod IS the apex, so there is no `prod.` label and no CNAMEs). # It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE — # the escape contract below reserves `$${...}` for expansions meant for a container's shell. # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames, @@ -606,7 +607,7 @@ def r_internet_egress(s: dict) -> dict: def r_httproutes(s: dict) -> list: """One HTTPRoute per ingressRoute, attaching host -> service port on the shared Gateway. - TLS is the Gateway's wildcard *.home.keithjackson.dev cert (no per-service cert step); + TLS is the Gateway's wildcard *.${BASE_DOMAIN} cert (no per-service cert step); ExternalDNS manages the record from the HTTPRoute (R-TLS/R-DNS, renderer-contract). """ out = [] diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 0592022..2f6c4cd 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -26,36 +26,49 @@ data: # authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns # (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM. # See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN - # subdomain, so the two clusters can no longer describe the same name. + # suffix, so the two clusters can no longer describe the same name. # - # dev -> home.keithjackson.dev (UNCHANGED — see below) - # prod -> prod.home.keithjackson.dev + # dev -> dev.home.keithjackson.dev (a subdomain; = this base default) + # prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label) # - # DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on - # `dev.home.keithjackson.dev` with apex CNAMEs `.home` -> `.prod.home`, but moving dev - # relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and - # the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2 - # (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as - # GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates - # it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre- - # tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths). - BASE_DOMAIN: "home.keithjackson.dev" + # OPERATOR DECISION 2026-09-04 (supersedes the interim split committed earlier the same day, which + # had prod on `prod.home.` with apex CNAMEs pointing at it). PROD IS THE APEX. That deletes the + # whole CNAME layer — `.home.keithjackson.dev` is prod's real name, not an alias — and it is + # DEV that carries a label. Convention is unchanged from GATEWAY_VIP / LONGHORN_DATA_PATH: the base + # default is the DEV value (dev restates it verbatim) and prod overrides it. What changed is which + # string dev holds. + # + # DEV'S VALUE IS A MIGRATION, NOT A VARIABLE CHANGE. The live dev cluster is serving the APEX right + # now: its wildcard cert, every public DNS record, its Keycloak issuer, ~12 OIDC clients' redirect + # URIs, kubectl SSO, and gitea.home.keithjackson.dev — which IS this repo's GitOps source AND the + # push target. Flipping this value on dev without first standing up the new host set STRANDS DEV: + # Flux's GitRepository url moves to gitea.dev.home.keithjackson.dev, which has no DNS record and no + # cert, so dev stops reconciling and cannot be fixed by pushing. ORDERED CUTOVER, operator present: + # infra/PROD-PROVISIONING.md §10. + # + # DEV MUST VACATE THE APEX BEFORE PROD PUBLISHES THERE. Both clusters describing + # `*.home.keithjackson.dev` at once is the 2026-09-04 incident precondition, re-created. + BASE_DOMAIN: "dev.home.keithjackson.dev" # ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind # EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered, # so it cannot be created, updated or deleted regardless of TXT ownership. # - # DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev` - # apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter - # that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching - # record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged - # from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically. + # BOTH ENVIRONMENTS USE THE CLOUDFLARE **ZONE**, `keithjackson.dev`. This value is NOT a per- + # environment isolation knob and must not be narrowed to either environment's BASE_DOMAIN. # - # PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE - # (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned - # chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED - # — it can never delete another environment's records — and the fallback is to set prod's value - # back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard. + # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17) — the Cloudflare provider uses --domain-filter for ZONE + # SELECTION as well as name filtering, so a filter that is a STRICT SUBDOMAIN of the hosted zone + # matches NO zone: ExternalDNS generates the record and then skips it with "no hosted zone matching + # record DNS Name", publishing NOTHING. This has been tried and reverted more than once (an agent + # narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in review). DO NOT "harden" + # it — the change is not defence in depth, it is a total outage of DNS publication. + # + # WHERE ISOLATION ACTUALLY COMES FROM, and it is sufficient: + # (1) the two environments request DISJOINT hostname sets (dev.home.… vs the apex), and + # (2) EXTERNALDNS_TXT_OWNER_ID differs, so neither registry ever claims the other's records. + # That pairing is what the 2026-09-04 incident needed; a domain filter cannot provide it, because + # both environments live in the one zone either way. EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev" # ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT: @@ -64,14 +77,35 @@ data: # `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on # one zone therefore garbage-collect each other. Each environment MUST have a distinct value. # - # DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare - # under owner `default`; changing dev's value would make dev's own external-dns stop recognising - # them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it. + # dev -> `dev` prod -> `prod` base default -> a non-colliding sentinel (below) # - # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that - # forgets this knob must not silently inherit dev's identity — that is precisely the incident. With - # this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a - # visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS. + # DEV IS `dev`, NOT `default` (changed 2026-09-04 with the move to dev.home.keithjackson.dev). Until + # this migration dev HAD to render `default`, because its LIVE apex records were already registered + # in Cloudflare's TXT registry under that owner and renaming would have orphaned every one of them. + # THAT CONSTRAINT NO LONGER BINDS: dev is publishing an ENTIRELY NEW record set under + # *.dev.home.keithjackson.dev, with no pre-existing registry entries to recognise, so the owner id + # is now free to be chosen on its merits. Two reasons to take `dev`: + # - `default` is THE VALUE YOU GET BY FORGETTING (it is the external-dns chart default, and both + # clusters silently shared it, which IS the incident). An environment whose identity is the + # forget-value can never be told apart from a misconfigured newcomer. No environment should + # hold it. + # - It makes vacating the apex an EXPLICIT, ORDERED, VERIFIABLE step. Dev's old apex records stay + # owned by `default`, which no environment now claims, so nothing deletes them implicitly on + # some future reconcile — they are retired deliberately, and prod's publish is gated on that + # retirement being confirmed in Cloudflare (infra/PROD-PROVISIONING.md §10) rather than on a + # race between two reconcile loops. + # + # PROD STAYS `prod` AND MUST NOT BECOME `default`, even though prod now serves the apex names that + # dev's `default`-owned records currently occupy. Giving prod `default` would hand prod ownership of + # the LIVE dev cluster's records while dev is still serving them, and under `policy: sync` prod + # would delete the ones no prod source claims. That is the 2026-09-04 incident, re-armed, with the + # roles swapped. Prod inherits the apex NAMES; it must never inherit the apex OWNER ID. + # + # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default` and not any real + # environment's id. A new environment that forgets this knob must not silently inherit another's + # identity — that is precisely the incident. With this sentinel such a cluster owns nothing, so it + # can DELETE nothing; it would publish under a visibly wrong owner id (grep the TXT records) + # instead of destroying a live environment's DNS. EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars" ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006) @@ -79,7 +113,7 @@ data: # ingress VIP of the `shared` Gateway. MUST equal the FIRST address of # ${CILIUM_LB_POOL_START}..${CILIUM_LB_POOL_STOP} below — the Gateway # takes the pool's first free IP, and coredns-custom answers - # *.home.keithjackson.dev with THIS so pod -> public-hostname traffic + # *.${BASE_DOMAIN} with THIS so pod -> public-hostname traffic # hairpins through the Gateway (correct SNI -> wildcard cert). # Base default = the DEV LAN VIP (was 10.73.0.240, an address on the # long-RETIRED libvirt network — corrected 2026-08-31 so an overlay that diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 1708eab..71f7533 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -5,6 +5,20 @@ # reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02). # It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git # on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1). +# +# ############################################################################################## +# ### *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) *** ### +# ### ### +# ### This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex ### +# ### `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url ### +# ### moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a ### +# ### merge to main IS the cutover — it executes on the live dev cluster with nobody present. ### +# ### Dev's Gitea is both its GitOps source and the push target: if the new host is not up ### +# ### with a Ready cert first, dev stops reconciling and pushing cannot repair it. ### +# ### ### +# ### Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator ### +# ### present. Merging is step D3, not step one. ### +# ############################################################################################## apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: @@ -131,7 +145,24 @@ patches: # via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO # secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and # revert this url (docs/05). - url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git + # + # *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. *** + # It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized — + # it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO + # postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and + # break the source outright (the L0 flux-postbuild-substitution check asserts exactly this). + # So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below. + # + # It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That + # makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization + # (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE + # DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already + # resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED + # BY PUSHING — the broken thing is the path Flux uses to read pushes. + # ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root + # Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for + # the migration window, and only restore the in-cluster url after the new host is proven. + url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git ref: branch: main # --- dev-scoped cluster-vars (R-ENV-02) --- @@ -148,33 +179,60 @@ patches: - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" - # --- DNS identity (R-DNS-01/02/03) ----------------------------------------------------- - # DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly - # like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running. - # Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its - # Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev - # (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to - # prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did - # not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration, - # not a variable change: infra/PROD-PROVISIONING.md §10. + # ========================================================================================== + # DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.** + # ========================================================================================== + # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day: + # + # dev -> dev.home.keithjackson.dev (this knob; = the base default, restated) + # prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label, + # and therefore no apex CNAME layer at all) + # + # *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right + # now. Merging this file does not perform the migration — it DEFINES the target. The cutover is + # an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10. + # + # WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20 + # lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps + # source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute, + # DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does + # not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED + # BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery + # is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair + # in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host + # FIRST, or park the source on the bootstrap Gitea for the window. + # + # ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh + # ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s + # server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's + # split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex. - op: replace path: /data/BASE_DOMAIN - value: "home.keithjackson.dev" - # ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself, - # and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped - # the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held. + value: "dev.home.keithjackson.dev" + # ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING + # F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a + # strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching + # record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it + # newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held. - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER value: "keithjackson.dev" - # DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT - # registry under owner `default` (the external-dns chart default, which is what both clusters - # silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records - # BY this string: any other value and it stops recognising every record it manages, then - # re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the - # whole point. The base default is a non-colliding sentinel, so this line is load-bearing. + # OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex + # records were already in Cloudflare's TXT registry under owner `default`, so renaming would + # have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix + # dev publishes a brand-new record set with no prior registry entries to recognise, so the id + # is chosen on its merits, and `default` has none — it is the value external-dns uses when you + # FORGET the flag, which is exactly how both clusters came to share it. Consequences, both + # deliberate: + # - dev's OLD apex records remain owned by `default`, which no environment now claims, so + # nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first + # publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing. + # - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev + # cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER. + # dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID - value: "default" + value: "dev" # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) ----------------------------- # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it # equals the base default. These values are what the LIVE dev cluster is running right now, so @@ -183,7 +241,7 @@ patches: # # Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN # (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers - # *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. + # *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. - op: replace path: /data/GATEWAY_VIP value: "192.168.1.53" diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index c7d7efc..21fb94a 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -129,14 +129,23 @@ patches: # # WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the # unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`). - # Two independent reasons, both consequences of the split-horizon design: - # 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's - # GitOps source lives inside the very cluster slated for decommission. Gitea must move to - # prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth. - # 2. SPLIT-HORIZON DNS SENDS IT TO THE WRONG PLACE. platform/coredns/coredns-custom.yaml - # answers the ENTIRE home.keithjackson.dev zone with ${GATEWAY_VIP}. On prod that is - # 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve - # gitea.home.keithjackson.dev to itself and never reach a repository. + # REASONING UPDATED 2026-09-04 for the new suffix direction (prod = the apex, dev = dev.home.), + # which changes WHY the blocker holds but not THAT it holds: + # 1. SELF-HOSTING CIRCULARITY — still true, and now the only reason. The Gitea this repo is + # pushed to runs ON THE DEV CLUSTER. Prod syncing from it would put prod's GitOps source + # inside the very cluster slated for decommission. Gitea must move to prod (or off-cluster) + # BEFORE dev is retired, or prod loses its source of truth. + # NOTE the suffix change makes the two Giteas DIFFERENT HOSTS: dev's is now + # gitea.dev.home.keithjackson.dev and prod's is gitea.home.keithjackson.dev. Prod pointing at + # the apex name therefore means "prod's OWN Gitea", never dev's — which is what makes the + # cutover below a one-line change once prod actually runs Gitea. + # 2. SPLIT-HORIZON DNS — NO LONGER A BLOCKER, and the note is kept because the old reasoning is + # easy to re-derive incorrectly. platform/coredns/coredns-custom.yaml answers the whole + # ${BASE_DOMAIN} zone with ${GATEWAY_VIP}; when prod's BASE_DOMAIN was `prod.home.` that meant + # prod resolved gitea.home.keithjackson.dev to PROD's gateway (192.168.2.20) where no Gitea + # listened — a misdirection. Now that prod IS the apex, resolving gitea.home.keithjackson.dev + # to prod's own Gateway is CORRECT: that is where prod's Gitea will be served from. What + # remains is purely reason 1 — the Gitea is not there YET. # # WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction): # bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the @@ -202,28 +211,41 @@ patches: path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" # ========================================================================================== - # PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident. + # PROD DNS IDENTITY (R-DNS-01/02/03) — PROD **IS** THE APEX. # ========================================================================================== - # PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to - # home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME - # public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one - # of which already happened: + # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day + # (prod on `prod.home.` behind apex CNAMEs). The simpler target, and the one authored here: + # + # dev -> dev.home.keithjackson.dev (dev carries the label; clusters/dev) + # prod -> home.keithjackson.dev (THE APEX — this knob) + # + # There is NO `prod.` label and NO CNAME layer. `.home.keithjackson.dev` is prod's REAL + # name, so nothing has to alias anything: prod's Gitea is gitea.home.keithjackson.dev, its + # Keycloak keycloak.home.keithjackson.dev, and the wildcard cert is `*.home.keithjackson.dev`. + # + # BACKGROUND — WHY THE SUFFIXES HAD TO SPLIT AT ALL. Until 2026-09-04 every hostname was + # HARDCODED to home.keithjackson.dev in the base manifests, so prod was authored to serve the + # EXACT SAME public names as the live dev cluster out of the SAME Cloudflare zone. Two + # consequences, one of which already happened: # 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit # default dev had) concluded it owned dev's records and DELETED EVERY ONE of them. # 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev` # wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would # have raced each other's `_acme-challenge` TXT records on that one zone. - # Moving prod to its own suffix dissolves both: the two clusters can no longer name the same - # record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here. # - # dev -> home.keithjackson.dev (UNCHANGED — dev is live; phase 2 moves it) - # prod -> prod.home.keithjackson.dev (this knob) - # - # CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is - # gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so - # on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make - # `.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they - # would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10. + # *** THE ORDERING CONSTRAINT THIS VALUE CREATES — READ BEFORE UNSUSPENDING ANYTHING. *** + # Prod now claims the very names the LIVE dev cluster is serving today. Both hazards above are + # therefore still armed until DEV HAS VACATED THE APEX. Prod's `platform` and its 10 downstream + # tiers are SUSPENDED for exactly this reason. The gate is not "review is done", it is: + # (a) dev is reconciling on *.dev.home.keithjackson.dev with a Ready wildcard cert, AND + # (b) dev's old apex A/CNAME records and their TXT registry entries are GONE from Cloudflare + # (verified against Cloudflare directly — a pfSense override is NOT evidence about public + # DNS; that mistake was made on 2026-09-04), AND + # (c) dev's old `*.home.keithjackson.dev` cert/Certificate is released, because prod ordering + # that same name set while dev's still exists counts against LE's duplicate-certificate + # limit (5 per week per identical name set). + # Full ordered runbook: infra/PROD-PROVISIONING.md §10. Unsuspending before (a)-(c) re-creates + # the 2026-09-04 incident with the roles reversed. # # [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml. # The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is @@ -231,30 +253,33 @@ patches: # trusts a different Keycloak than the one prod actually runs. - op: replace path: /data/BASE_DOMAIN - value: "prod.home.keithjackson.dev" - # SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's - # OWN subtree, so prod cannot create, update or delete any record outside it no matter what the - # TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for - # ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped - # the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no - # matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's - # records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on - # EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own. + value: "home.keithjackson.dev" - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER - # MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG, - # 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the - # record then skipped it with "no hosted zone matching record DNS Name", so prod would - # publish NOTHING. Isolation from dev does NOT come from this filter — it comes from - # (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and - # (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records. + # MUST be the Cloudflare ZONE. Prod now serves the apex, so `home.keithjackson.dev` looks + # like a tighter, more honest filter than the zone — IT IS NOT, IT IS AN OUTAGE. FINDING + # F-DNS1 (DECISIONS-LOG 2026-08-17) proved a STRICT SUBDOMAIN of the hosted zone drops the + # zone entirely: ExternalDNS generates the record then skips it with "no hosted zone matching + # record DNS Name", so prod publishes NOTHING. An agent narrowed this to + # prod.home.keithjackson.dev on 2026-09-04 and it was reverted in review; do not retry it. + # Isolation from dev does NOT come from this filter — it comes from + # (1) the two clusters requesting DISJOINT hostname sets (apex vs dev.home.), and + # (2) txtOwnerId: prod vs dev, so neither registry ever claims the other's records. # That pairing is what the 2026-09-04 incident needed and this filter cannot provide. value: "keithjackson.dev" # SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so # both inherited the chart default `default`; prod then treated dev's TXT-registry entries as - # its own and, under policy: sync, deleted the records behind them. This value MUST differ from - # dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and - # cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix. + # its own and, under policy: sync, deleted the records behind them. + # + # PROD MUST NOT BE GIVEN `default`, however tempting it now looks. Prod is taking over the apex + # NAMES, and dev's existing apex records are registered under owner `default` — so `default` is + # the one value that would let prod adopt them. It would also let prod SYNC-DELETE the ones no + # prod source claims, on a cluster dev is still serving. Prod inherits the apex names; it must + # never inherit the apex OWNER ID. `prod` keeps prod fail-safe: records owned by `default` are + # invisible to it, so an early unsuspend cannot destroy dev's DNS the way 2026-09-04 did — it + # simply fails to publish until those records are retired, which is the correct behaviour. + # dev is `dev` (clusters/dev, changed with dev's migration); the base default is a + # non-colliding sentinel. All three differ. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID value: "prod" diff --git a/infra/PROD-PROVISIONING.md b/infra/PROD-PROVISIONING.md index 2e7a9e4..681bf88 100644 --- a/infra/PROD-PROVISIONING.md +++ b/infra/PROD-PROVISIONING.md @@ -1172,7 +1172,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be | `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address | | `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address | | `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target | -| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. | +| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real. Prod's value was inverted the same day to the APEX `home.keithjackson.dev`, with dev moving to `dev.home.keithjackson.dev` — see §10a, which is authored but NOT yet applied. | - [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both* overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the @@ -1283,12 +1283,20 @@ single-server-etcd window, and avoids a later member-add dance. specifically. - [ ] Monitoring and alerts live on prod. - [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03). - Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2. -- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10): - distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and - prod's own Keycloak realm + per-service client secrets. + Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10a + "The L2 tests hardcode the apex". +- [ ] **Prod serves the apex DNS/TLS identity** and dev's records are provably untouched (§10/§10a): + distinct `EXTERNALDNS_TXT_OWNER_ID` (`prod` vs `dev`), prod wildcard cert + `*.home.keithjackson.dev`, dev migrated to `*.dev.home.keithjackson.dev` with its old apex + records retired (§10a step D8), and prod's own Keycloak realm + per-service client secrets. -## 10. DNS/TLS environment split — **the 2026-09-04 incident, and phase 1 of the fix** +## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture** + +> **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's +> `platform` tier and its 10 downstream tiers are SUSPENDED. Dev is **live on the apex**. The +> ordered cutover is **§10a**, and merging the authored change to `main` is step **D3** of it, +> not step one — dev self-reconciles `main`, so a merge outside the runbook window executes +> the migration unattended. ### What happened @@ -1320,43 +1328,82 @@ values; for DNS and TLS that promise was false, so `clusters/prod` was *authored public identity. Two clusters that describe the same DNS name will fight over it no matter how the DNS controller is configured. -### The architecture decision (operator) +### The architecture decision (operator) — **INVERTED 2026-09-04, this is the current target** ``` -dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev -prod -> *.prod.home.keithjackson.dev gitea.prod.home.keithjackson.dev -apex -> CNAME .home.keithjackson.dev -> .prod.home.keithjackson.dev +dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev +prod -> *.home.keithjackson.dev gitea.home.keithjackson.dev (THE APEX) ``` -### PHASE 1 — **prod only. Dev did not move.** (this change) +**Prod *is* the apex. There is no `prod.` label and no CNAME layer at all** — +`.home.keithjackson.dev` is prod's real name, not an alias for one. This **supersedes** the +interim split committed earlier the same day (`c8c1d29`), which put prod on +`*.prod.home.keithjackson.dev` behind apex CNAMEs pointing at it. That layer is deleted: fewer +moving parts, no CNAME chain to keep in step, and prod ends up on the names users already type. -`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it. +The cost of the inversion is that **dev must move**, and dev is the live cluster. The interim split +was chosen precisely because it moved only the unbuilt environment; this one does not have that +property, so the whole of §10's remaining risk now sits on dev's side, and it is sequenced below. -| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | -|---|---|---|---| -| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | -| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` | -| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` | +### The values, per environment -**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod -alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken -while the operator is away. Dev's rendered output is **byte-identical** except for one added line, -`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified -by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for -both overlays and diffing. +| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | Ansible `k3s_base_domain` | +|---|---|---|---|---| +| base default | `dev.home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | `dev.home.keithjackson.dev` (role default) | +| **dev** | `dev.home.keithjackson.dev` | `dev` | `keithjackson.dev` | (inherits the role default) | +| **prod** | `home.keithjackson.dev` | `prod` | `keithjackson.dev` | `home.keithjackson.dev` (`group_vars/k3s_prod`) | -**Two independent guards now, either sufficient alone:** +`[parity]`: `k3s_base_domain` must equal the same cluster's `BASE_DOMAIN`. The kube-apiserver's +`oidc-issuer-url` is a k3s **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux +post-build substitution can never reach it — hence one fact in two places. If they disagree the +apiserver validates tokens against a different Keycloak than the cluster runs: kubectl SSO fails +while the cert-based admin kubeconfig keeps working, so it presents as "SSO is broken", not as skew. -1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is - the direct fix for the incident. Dev **must** render exactly `default`: its live records are - already registered under that owner, and any other value would make dev's own external-dns stop - recognising them and re-create duplicates beside the orphans. -2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree. - Defence in depth, added because the first guard is a single string in a single file. +**`EXTERNALDNS_DOMAIN_FILTER` is the Cloudflare ZONE for BOTH, and must stay that way.** FINDING +F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for **zone selection** as +well as record matching, so a filter that is a strict subdomain of the hosted zone matches **no +zone** — external-dns generates the record and then skips it with *"no hosted zone matching record +DNS Name"*, publishing **nothing**. This has been "hardened" and reverted more than once. It is not +defence in depth; it is a total outage of DNS publication. Isolation comes from the two guards below, +which a domain filter cannot provide anyway: both environments live in the one zone regardless. -The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a -future environment that forgets the knob must not silently inherit dev's identity — that is exactly -what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing. +**The two guards that actually work:** + +1. **`txtOwnerId`, per-environment identity** — a cluster only ever reconciles records its own owner + id claims in the TXT registry. This is the direct fix for the incident. +2. **Disjoint hostname sets** — after the migration the two clusters never describe the same record, + so there is nothing to fight over. *This guard does not exist during the migration window*, which + is the entire reason the ordering below is mandatory. + +#### Why dev's owner id changed from `default` to `dev` + +Until this migration dev **had** to render `default`: its live apex records were already registered +in Cloudflare's TXT registry under that owner, and renaming would have orphaned every one of them. +**The migration dissolves that constraint** — on `*.dev.home.keithjackson.dev` dev publishes an +entirely new record set with no prior registry entries to recognise, so the id is free and is chosen +on its merits. `default` has none: it is the value external-dns uses when you **forget** the flag, +which is exactly how both clusters came to share it, and an environment whose identity is the +forget-value can never be told apart from a misconfigured newcomer. Consequences, both deliberate: + +- Dev's **old apex records stay owned by `default`**, which no environment now claims, so nothing + deletes them implicitly on some future reconcile. Retiring them is an explicit step (**D7**) and + prod's first publish is gated on that retirement being *confirmed* — not on two reconcile loops + racing each other. +- **Prod stays `prod` and must never be given `default`.** Prod is taking over the apex *names*, and + dev's existing apex records are registered under `default` — so `default` is the one value that + would let prod adopt them, and under `policy: sync` delete the ones no prod source claims, on a + cluster dev is still serving. That is the 2026-09-04 incident re-armed with the roles swapped. + Prod inherits the apex **names**; it must never inherit the apex **owner id**. + +All three values differ, and the base default remains a sentinel so an environment that forgets the +knob owns nothing and can therefore delete nothing. + +### PHASE 1 — prod tokenized and moved off dev's names (`c8c1d29`, done; suffix since inverted) + +`BASE_DOMAIN` was a **dead knob** — declared in `clusters/base/cluster-vars.yaml` and referenced by +no reconciled manifest, because every hostname was a hardcoded literal. `c8c1d29` made it real (141 +references across the reconciled set) and gave prod its own suffix. That work stands; only the +*value* prod holds has changed, from `prod.home.keithjackson.dev` to the apex. ### What could NOT be tokenized, and why @@ -1365,70 +1412,195 @@ what happened here. With the sentinel such a cluster owns nothing, so it can del Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and **cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately. - **Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak. - Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client - secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret - is per-realm. + **Both environments now need action:** dev's copy must move to `keycloak.dev.home.keithjackson.dev` + as part of step **D5**, and prod needs its own copy with `keycloak.home.keithjackson.dev` and a + client secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client + secret is realm-scoped. - **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s - **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution - can never reach it. Correctly tokenized on the **Ansible** side instead: new role default - `k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in - `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`. - `[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver - validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the - cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew). + server flag, not a manifest. Handled on the **Ansible** seam instead: role default + `k3s_base_domain` (dev's value) + derived `k3s_oidc_issuer_url`, overridden for prod in + `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`, + so a fourth prod server inherits it from group membership. Verified per host by a read-only probe + (`connection: local`, zero SSH, `changed=0`): all three dev servers resolve + `https://keycloak.dev.home.keithjackson.dev/realms/homelab`, all three prod servers resolve + `https://keycloak.home.keithjackson.dev/realms/homelab`. +- **`spec.url` of dev's `GitRepository`** (`clusters/dev/kustomization.yaml`) — see the strand below. + It *cannot* be tokenized: the `flux-system` root Kustomization that renders `./clusters/dev` + declares no `postBuild.substituteFrom`, so `gitea.${BASE_DOMAIN}` there would ship the **literal + token** and break the source outright (the L0 `flux-postbuild-substitution` check asserts exactly + this). It is a hand-maintained literal that must move in lockstep with `BASE_DOMAIN`. -### A pre-existing prod problem this incidentally FIXED +--- -§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC -issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN, -making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on -its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by -**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer -and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's -external-dns has actually published the record; until then use the break-glass cert kubeconfig. +## 10a. THE MIGRATION RUNBOOK — ordering is load-bearing -### Verify on prod's first reconcile +> **Prod's `platform` tier and its 10 downstream tiers are SUSPENDED and must stay suspended until +> step P1.** Nothing below is safe to run out of order, and nothing below should be run while the +> operator is away. -- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup - as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone - ("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it - will create **nothing** — fail-closed, never destructive. Fallback: set prod's - `EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is - sufficient on its own. -- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching - dev's `_acme-challenge` records. -- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry - entries still read owner `default` and that prod's read owner `prod`. -- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client - secret is realm-scoped; prod cannot reuse dev's). +### The two ways this goes wrong -### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**) +**(1) BOTH CLUSTERS ON THE APEX — the 2026-09-04 incident, recreated.** If prod publishes +`*.home.keithjackson.dev` before dev has vacated it, two clusters describe the same records in the +same zone again. Distinct `txtOwnerId`s stop prod from *deleting* dev's records — that guard holds — +but they do not stop the second failure mode: both clusters requesting a `letsencrypt-prod` wildcard +for the **same name set** and racing each other's `_acme-challenge` TXT records, so neither reliably +solves DNS-01. **Rule: dev vacates the apex first, and the vacate is verified against Cloudflare +before prod is unsuspended.** -Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the -Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change. -Required, roughly in order: +**(2) THE STRAND — dev's GitOps source is the thing that moves.** `spec.url` of dev's +`GitRepository` is `https://gitea.home.keithjackson.dev/...`, which is **both** dev's GitOps source +**and** the PM's push target. Flipping dev's `BASE_DOMAIN` moves Gitea's HTTPRoute, its public DNS +record and the wildcard cert covering it to `gitea.dev.home.keithjackson.dev` all at once. If that +host does not already resolve and serve a valid, Ready cert at that instant, **dev stops reconciling +and cannot be fixed by pushing** — the broken thing *is* the path Flux uses to read pushes. Recovery +is then hands-on and needs LAN access to the cluster. -1. **`BASE_DOMAIN` → `dev.home.keithjackson.dev`** in `clusters/dev`, plus `k3s_base_domain` for the - dev inventory group. -2. **A new `*.dev.home.keithjackson.dev` wildcard cert** issued and Ready *before* cutover. -3. **Keycloak issuer change.** The realm's issuer URL is baked into every token; every OIDC client - (`grafana`, `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, - `qbittorrent`, `radarr`, `sonarr`, and the `kubernetes` public client) needs its redirect URIs - updated in the same change, and the k3s apiserver `oidc-issuer-url` with it — which means a - **rolling k3s restart** on the dev servers. -4. **kubectl SSO re-issue** for every user (issuer changed). -5. **`gitea.dev.home.keithjackson.dev`** — this is the GitOps source URL in - `clusters/dev/kustomization.yaml`. Flux must be able to resolve and trust the new host *before* - the URL is switched, or dev stops reconciling and the change cannot be pushed to fix itself. - Sequence this last, and keep the bootstrap Gitea available as the recovery path (docs/05). -6. **`EXTERNALDNS_TXT_OWNER_ID` for dev.** Once dev's names move, its existing `default`-owned - records are for names dev no longer serves; retire them deliberately rather than letting a - renamed owner orphan them. -7. **Apex CNAMEs** `.home.keithjackson.dev` → `.prod.home.keithjackson.dev`. This is what - actually **moves user traffic to prod**, and it must not land before prod serves those services. -8. **The L2 behavioural conformance tests hardcode `home.keithjackson.dev`** (`dns-exposure`, - `dns-internal-resolution`, `gateway-ingress-reachable`, `oidc-sso-grafana`, `sso-forward-auth`, - `trusted-tls`). They are test inputs, not reconciled manifests, so they were left alone in phase 1 - — but they will fail against prod and must be parameterized before §9's "suite passes green - against the prod cluster" can be claimed. +**(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root +Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main` +too. **A merge to `main` therefore *executes* this migration on the live dev cluster, unattended.** +The authored change is a *target*, not a deployment. **Do not merge these files to `main` outside the +runbook window.** Park them on a branch until the operator is at the keyboard and step **D1** has +been done. + +### D — dev vacates the apex (operator present, in this order) + +- [ ] **D0. Snapshot the escape routes before touching anything.** Confirm a working break-glass + **cert-based** admin kubeconfig for dev (not the OIDC one — OIDC breaks in D6). Confirm the + retained bootstrap Gitea can be started (`bootstrap/gitea/bootstrap-gitea.sh`; dev's historical + recovery host was `192.168.2.103:3000`, prod's current bootstrap is `192.168.2.132:3000`). + Record dev's current Cloudflare record set (names + TXT registry owners) so D7 has a checklist + and so an accidental deletion is detectable. +- [ ] **D1. Break the automatic trigger.** Either `flux suspend kustomization flux-system -n + flux-system` on dev (stops the root from self-applying the merge), **or** re-point dev's + `GitRepository` url at the bootstrap Gitea for the window. Do this **before** the merge, not + after. This is what converts "merging strands dev" into "merging is inert until you say go". +- [ ] **D2. Publish and prove the new host set *while dev still serves the apex*.** The new names + must exist before anything depends on them: + - a `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev` **wildcard Certificate issued + and `Ready`** (LE prod, DNS-01) — see the ACME note below before ordering it; + - public DNS for `gitea.dev.home.keithjackson.dev` resolving to dev's Gateway VIP + (192.168.1.53), **checked against Cloudflare / a public resolver, never through the pfSense + override** — that mistake was made on 2026-09-04 and produced a false healthy reading; + - `curl -sS https://gitea.dev.home.keithjackson.dev/api/v1/version` succeeding with a + publicly-trusted cert from a host on the dev LAN. + Until D2 is green, D3 is not safe. +- [ ] **D3. Merge the authored change to `main`** (`clusters/base/cluster-vars.yaml`, + `clusters/dev/`, `clusters/prod/`, the two Ansible files, `platform/externaldns`, + `catalog/render.py`). With D1 in place nothing moves yet. +- [ ] **D4. Release dev's reconciliation deliberately.** Resume the root Kustomization (or restore + the in-cluster url, now pointing at `gitea.dev.home.keithjackson.dev`) and watch + `flux get kustomizations -A` converge. Dev's Gateway listeners, HTTPRoutes, coredns-custom zone + and Homepage links all move here. Dev's external-dns publishes the new record set under owner + `dev`. +- [ ] **D5. Keycloak: issuer + every client.** Dev's realm issuer becomes + `https://keycloak.dev.home.keithjackson.dev/realms/homelab`. The issuer is baked into every + token, and **~12 OIDC clients** need their redirect URIs moved in the same change: `grafana`, + `gitea`, `nextcloud`, `semaphore`, `headlamp`, `crafty`, `pihole`, `jackett`, `qbittorrent`, + `radarr`, `sonarr`, plus the `kubernetes` public client. Most are reconciled from + `platform/keycloak/keycloak-realm-import.yaml` and move with D4; **headlamp's is the exception** + — its issuer/callback live in a SOPS-encrypted Secret and must be re-encrypted by hand. +- [ ] **D6. Rolling k3s restart + kubectl SSO re-issue.** `k3s_base_domain` now renders a different + `oidc-issuer-url` into `/etc/rancher/k3s/config.yaml`, so the `k3s_server` role is **no longer + a no-op on dev**: it rewrites the file and fires the Restart handler on all three servers. Run + it **one server at a time**, confirming `kubectl get nodes` between each. Every user's OIDC + kubeconfig is invalidated and must be re-issued; the cert-based admin kubeconfig keeps working + throughout, which is what makes this step recoverable. **Do not reboot nodes** — restart the + `k3s` service only. +- [ ] **D7. Retire dev's old apex records.** They are owned by `default`, which no environment now + claims, so nothing removes them automatically. Delete the `.home.keithjackson.dev` A + records, the apex record, **and their TXT registry entries** (`external-dns-*` TXT), plus any + stale `_acme-challenge.home.keithjackson.dev`. *Alternative, if you prefer external-dns to do + its own cleanup:* set dev's `EXTERNALDNS_TXT_OWNER_ID` back to `default` for exactly one + reconcile after D4 — `policy: sync` will then garbage-collect the apex set it no longer has + sources for — and only then switch it to `dev`. That is two commits and needs watching; the + direct deletion is preferred because it is verifiable in one place. +- [ ] **D8. Verify the apex is EMPTY, against Cloudflare.** `dig +short .home.keithjackson.dev + @1.1.1.1` returns nothing for every service name, and no TXT registry entry for those names + remains. **This is the gate on the whole P sequence.** A pfSense/PiHole override is not + evidence about public DNS. + +### P — prod takes the apex + +- [ ] **P1. Only now unsuspend prod**, in dependency order: `flux resume kustomization platform`, + then the 10 downstream tiers. Confirm D8 first, in Cloudflare, not from a local resolver. +- [ ] **P2. Prod's external-dns publishes under owner `prod`.** Check the logs for a matching hosted + zone (F-DNS1 fail-closed mode: no zone selected ⇒ nothing created — which is safe, but means + the filter was narrowed against instruction). Confirm the new TXT registry entries read owner + `prod` and that no record owned by `dev` was touched. +- [ ] **P3. Prod's wildcard cert `*.home.keithjackson.dev` + `home.keithjackson.dev` is `Ready`** and + solved DNS-01 without colliding with any `_acme-challenge` dev still holds. +- [ ] **P4. Prod's kubectl SSO works without an `/etc/hosts` pin.** Prod nodes resolve + `keycloak.home.keithjackson.dev` through **public** upstreams (§4), so this only becomes true + once P2 has published that name at prod's Gateway VIP (192.168.2.20). Before then a prod node + resolving its own issuer reaches **dev**. Use the break-glass cert kubeconfig until P2 is done. +- [ ] **P5. Cut prod's GitOps source over** from the workstation bootstrap Gitea + (`http://192.168.2.132:3000/...`) to `https://gitea.home.keithjackson.dev/...` once prod runs + its own Gitea — the block in `clusters/prod/kustomization.yaml` owns that procedure. Note the + inversion **simplified** this: prod's apex Gitea name and dev's `gitea.dev.home...` are now + different hosts, so prod pointing at the apex means "prod's own Gitea", never dev's. + +### ACME: the duplicate-certificate limit + +Let's Encrypt rate-limits **duplicate certificates — 5 per week for an identical set of names.** +Prod's wildcard is `*.home.keithjackson.dev` + `home.keithjackson.dev`, which is **exactly the name +set dev holds today**. Every prod order therefore counts against the same bucket as dev's historical +renewals of that pair, and a retry loop during P3 can exhaust it and leave prod with **no trusted +cert for up to a week**. Mitigations: + +- Order prod's cert **after** D8, so dev is no longer renewing the same pair. +- Let dev's old `*.home.keithjackson.dev` `Certificate`/Secret be **deleted** in D4/D7 rather than + left to renew in the background. +- Rehearse against **letsencrypt-staging** first (`ACME_CLUSTER_ISSUER`) — staging has its own, + far looser limits — and only switch prod to `letsencrypt-prod` when the DNS-01 solve is proven. +- Do **not** delete-and-recreate prod's `Certificate` to "retry"; each recreation is a fresh order. + +### The L2 tests hardcode the apex + +Six behavioural (L2) chainsaw tests embed `home.keithjackson.dev` as a literal: + +| test | file | +|---|---| +| `dns-exposure` | `conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml` | +| `dns-internal-resolution` | `conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml` | +| `gateway-ingress-reachable` | `conformance/tests/l2-behavioral/gateway-ingress-reachable/chainsaw-test.yaml` (+ `fixtures/app-and-probe.yaml`) | +| `oidc-sso-grafana` | `conformance/tests/l2-behavioral/oidc-sso-grafana/chainsaw-test.yaml` | +| `sso-forward-auth` | `conformance/tests/l2-behavioral/sso-forward-auth/chainsaw-test.yaml` | +| `trusted-tls` | `conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml` | + +They are **test inputs, not reconciled manifests**, so they are deliberately left alone by the +authoring change: they are the only thing still asserting the apex, and while dev is live on the apex +they are *correct*. Under the new direction the literal happens to become right for **prod** and +wrong for **dev**, so they must be **parameterized per environment** (not merely edited) before +either §9's *"suite passes green against the prod cluster"* or a green L2 run against migrated dev +can be claimed. Until then, expect the dev L2 suite to fail between D4 and that parameterization — +a known, scheduled failure, not a regression. + +The L1 *rendering* tests need no change: their goldens carry `${BASE_DOMAIN}` as a token, and the +literal hostnames in them come from each test's own fixture `ingressRoutes[].host`, which is +environment-independent by construction. + +### Offline proof carried out for the authoring change (no cluster touched) + +All 12 `substituteFrom` paths were rendered for **both** overlays through the real toolchain +(`kubectl kustomize --load-restrictor LoadRestrictionsNone | flux envsubst --strict`, +with each overlay's effective `cluster-vars` taken from `kubectl kustomize clusters/`), before +and after the change. + +- **Dev's diff is *only* the hostname move.** Proven by inverse substitution rather than by reading + the diff: applying `s/dev.home.keithjackson.dev/home.keithjackson.dev/` to the new render restores + the baseline **byte-for-byte** in all 12 paths, plus 2 declared `txtOwnerId: default → dev` lines. + 145 changed line pairs, all classified: 143 hostname-move, 2 declared owner-id, **0 unexplained**, + and **0 bare-apex remnants**. The 2 paths with no change (`cilium`, `velero`) provably contain no + hostname at all (`grep -c keithjackson.dev` = 0). +- **Prod lands on the bare apex:** **0** occurrences of `prod.home.keithjackson.dev` and **0** of + `dev.home.keithjackson.dev` anywhere in prod's rendered output; 143 apex names. +- **Certs:** dev `*.dev.home.keithjackson.dev` + `dev.home.keithjackson.dev`; prod + `*.home.keithjackson.dev` + `home.keithjackson.dev`. +- **Positive controls** (the harness is provably not blind): perturbing an unrelated knob + (`GATEWAY_VIP`) surfaces 1 unexplained line; perturbing `BASE_DOMAIN` to a wrong value fails 10 of + 12 paths; injecting a single un-moved hostname is caught by the independent bare-apex scan (which + the byte-identity check alone cannot see — which is why both exist). +- Gates: `ci/l0.sh`, `ci/l1.sh`, `ci/render-golden.sh`, `ci/lint.sh`, + `flux-postbuild-substitution/check.sh` (254 assertions), `catalog/render.py --selftest`, + `ci/spec-immutability.sh` — all **PASS**. diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index 056ae85..6619c99 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -293,35 +293,47 @@ node_baseline_dns_servers: # k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through # these PUBLIC upstreams — not through CoreDNS's split-horizon answer. # - # THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was - # authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev - # publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so - # kubectl OIDC SSO on prod could not work at all. Now prod's issuer is - # keycloak.prod.home.keithjackson.dev, and the public record for that name is published by - # PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer - # and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed. - # - # STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's - # external-dns has actually published the record. Until then the issuer does not resolve - # from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass - # path, and it always works. Ordering, not a design gap. + # ORDERING HAZARD, NOT YET CLEARED. Prod's issuer is keycloak.home.keithjackson.dev — an APEX + # name, because prod IS the apex under the 2026-09-04 inversion (k3s_base_domain below). + # That is the name the LIVE DEV CLUSTER answers today, and its public A record currently + # points at DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. A prod + # node resolving its own issuer through these public upstreams therefore reaches dev until: + # (1) dev has migrated to *.dev.home.keithjackson.dev and RETIRED its apex records, and + # (2) PROD's own external-dns has published keycloak.home.keithjackson.dev at PROD's + # Gateway VIP (192.168.2.20). + # Only then do the public answer and the in-cluster split-horizon answer AGREE, and no + # /etc/hosts pin is needed. Until then kubectl OIDC SSO on prod cannot work; the cert-based + # admin kubeconfig is the break-glass path and always works. Ordering, not a design gap — + # the ordered steps are infra/PROD-PROVISIONING.md §10. node_baseline_dns_search: [] # --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) -------------------- -# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/ -# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and -# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a -# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so -# postBuild.substituteFrom never sees it. Full rationale in -# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this. +# PROD **IS** THE APEX. [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml — +# the same one-fact-in-two-places contract as k3s_data_dir and k3s_api_host. The Flux side +# cannot supply this: the kube-apiserver's oidc-issuer-url is a k3s server FLAG written to +# /etc/rancher/k3s/config.yaml, not a manifest, so postBuild.substituteFrom never sees it. +# Full rationale in roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url derives. # -# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the -# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns -# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE); -# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is -# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client -# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against. -k3s_base_domain: "prod.home.keithjackson.dev" +# WHY THE SUFFIXES SPLIT (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve +# the SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns +# deleted every one of dev's public records. +# +# WHICH WAY ROUND — INVERTED 2026-09-04 by operator decision, superseding the interim split +# committed earlier the same day (prod on `prod.home.` behind apex CNAMEs). The simpler target: +# prod -> home.keithjackson.dev (THE APEX; this variable — no `prod.` label, no CNAMEs) +# dev -> dev.home.keithjackson.dev (roles/k3s_server/defaults/main.yml) +# So prod's Keycloak issuer is https://keycloak.home.keithjackson.dev/realms/homelab, and the +# `kubernetes` OIDC client in PROD's OWN Keycloak realm is what prod's kubectl SSO authenticates +# against. +# +# ORDERING HAZARD SPECIFIC TO THIS VARIABLE. keycloak.home.keithjackson.dev is the name the LIVE +# dev cluster answers today, and prod's nodes resolve the issuer through PUBLIC upstreams (see +# node_baseline_dns_search above). So until dev has vacated the apex and PROD's external-dns has +# published that name at PROD's Gateway VIP (192.168.2.20), a prod node resolving its own issuer +# gets DEV's VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. kubectl OIDC SSO on prod +# therefore does not work until §10's dev-vacate steps are complete; the cert-based admin +# kubeconfig is the break-glass path and always works. Ordering, not a design gap. +k3s_base_domain: "home.keithjackson.dev" # --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) ----- # roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml index d052d12..9df4912 100644 --- a/infra/ansible/roles/k3s_server/defaults/main.yml +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -84,15 +84,26 @@ k3s_tls_sans: >- # while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken", # not like a config skew. The break-glass cert kubeconfig is why this is recoverable. # -# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same -# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the -# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte -# what it already is and the Restart handler does NOT fire on a re-run. +# THE DEFAULT IS THE DEV VALUE — the same convention as k3s_data_dir above and cluster-vars' +# GATEWAY_VIP / LONGHORN_DATA_PATH: the role default is dev's value and prod overrides it in +# inventory/group_vars/k3s_prod/vars.yml (the same inventory seam k3s_data_dir / k3s_api_host / +# kube_vip_address use, so a FOURTH prod server inherits it from group membership rather than +# from anyone remembering). # -# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam -# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it -# from group membership rather than from anyone remembering. -k3s_base_domain: "home.keithjackson.dev" +# *** WHICH DEV VALUE — UPDATED 2026-09-04, AND IT IS AHEAD OF THE LIVE DEV CLUSTER. *** +# The operator inverted the suffix split: PROD is now the apex `home.keithjackson.dev` and DEV +# carries the label `dev.home.keithjackson.dev`. This default therefore moved from the apex to +# dev's subdomain, mirroring BASE_DOMAIN in clusters/dev/kustomization.yaml. +# +# CONSEQUENCE — this default no longer matches what the live dev nodes are running, so unlike +# every previous edit to this file A DEV RE-RUN IS NOT A NO-OP. Rendering config.yaml with the +# new issuer CHANGES the file, FIRES the Restart handler, and rolls k3s on all three dev +# servers, which also invalidates every user's OIDC kubeconfig (SSO must be re-issued). That is +# a deliberate step of the dev DNS migration, NOT a side effect to discover during unrelated +# work: DO NOT run the k3s_server role against dev until step D6 of +# infra/PROD-PROVISIONING.md §10, and never while the operator is away. The break-glass +# cert-based admin kubeconfig keeps working throughout, which is what makes the step safe. +k3s_base_domain: "dev.home.keithjackson.dev" # Derived, so the realm path is stated once. Override k3s_base_domain, not this. k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab" diff --git a/platform/externaldns/helmrelease.yaml b/platform/externaldns/helmrelease.yaml index 3a56c3b..4324681 100644 --- a/platform/externaldns/helmrelease.yaml +++ b/platform/externaldns/helmrelease.yaml @@ -39,24 +39,33 @@ spec: # garbage-collected them. `policy: sync` is correct and stays; what was missing was identity. # Full write-up: infra/PROD-PROVISIONING.md §10. # - # TWO INDEPENDENT GUARDS NOW, either of which alone prevents a recurrence: - # 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns. - # 2. domainFilters — per-environment name scope; prod cannot even see names outside its - # own subtree. Defence in depth, added after the incident. + # THE TWO GUARDS THAT ACTUALLY PREVENT A RECURRENCE — note that neither is domainFilters: + # 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns. + # 2. DISJOINT HOSTNAME SETS — the two environments serve different suffixes, so they never + # describe the same record in the first place. dev = *.dev.home.keithjackson.dev, + # prod = *.home.keithjackson.dev (prod IS the apex; operator decision 2026-09-04). # --------------------------------------------------------------------------------------------- - # Which names this instance may manage at all. Cloudflare-provider caveat, empirically hit - # 2026-08: this value is used for ZONE selection as well as record matching, and a filter that is - # a strict subdomain of the zone dropped the parent zone entirely ("no hosted zone matching record - # DNS Name") — so DEV keeps the zone apex `keithjackson.dev`, which is also what dev serves the - # apex of. PROD narrows to `prod.home.keithjackson.dev`. If prod's first reconcile logs no - # matching zone, that caveat still applies: prod creates NOTHING (fail-closed, never destructive) - # and the fallback is to widen prod's EXTERNALDNS_DOMAIN_FILTER back to the zone apex, leaving - # txtOwnerId as the sole guard. See clusters/base/cluster-vars.yaml. + # Which names this instance may manage at all. BOTH ENVIRONMENTS RENDER THE CLOUDFLARE **ZONE** + # `keithjackson.dev` HERE, and this is NOT an oversight to be tightened. + # + # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for ZONE + # SELECTION as well as record matching, so a filter that is a STRICT SUBDOMAIN of the hosted zone + # matches NO zone — external-dns generates the record and then skips it with "no hosted zone + # matching record DNS Name", publishing NOTHING. This has been "improved" and reverted more than + # once (an agent narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in + # review). It is not defence in depth; it is a total outage of DNS publication. Isolation comes + # from the two guards above, which a domain filter cannot provide anyway — both environments live + # in the ONE zone regardless of what this says. See clusters/base/cluster-vars.yaml. domainFilters: - "${EXTERNALDNS_DOMAIN_FILTER}" # The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence - # caused the incident. Dev renders `default` (its live records are already registered under that - # owner; changing it would orphan them and duplicate every record). Prod renders `prod`. + # caused the incident. dev renders `dev`, prod renders `prod`, and the BASE default is a + # non-colliding sentinel so a new environment that forgets the knob owns nothing and can delete + # nothing. NOTHING renders `default` any more (the external-dns chart default, and the value both + # clusters silently shared): dev held it only because its live apex records were registered under + # it, and dev's move to *.dev.home.keithjackson.dev retired that constraint along with those + # records. Prod must NOT be given `default` even though it inherits the apex NAMES — that would + # hand prod ownership of dev's old apex records and let `policy: sync` below delete them. txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}" policy: sync # create + update + delete records as services come and go — but ONLY # records this owner id owns AND this domainFilter admits (see above).