Revert "feat(dns): INVERT to operator's simpler layout — dev to subdomain, prod to apex"

This reverts commit 9137bbd0c3.

catalog/render.py

@@ -47,8 +47,7 @@ LBL = "homelab.keithjackson.dev"
 # The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal.
 # Every rendered manifest lands under a Flux path whose Kustomization declares
 # postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at
-# reconcile time: dev renders dev.home.keithjackson.dev, prod renders home.keithjackson.dev (the
-# APEX — operator decision 2026-09-04; prod IS the apex, so there is no `prod.` label and no CNAMEs).
+# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev.
 # It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE —
 # the escape contract below reserves `$${...}` for expansions meant for a container's shell.
 # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames,
@@ -607,7 +606,7 @@ def r_internet_egress(s: dict) -> dict:
 def r_httproutes(s: dict) -> list:
     """One HTTPRoute per ingressRoute, attaching host -> service port on the shared Gateway.
 
-    TLS is the Gateway's wildcard *.${BASE_DOMAIN} cert (no per-service cert step);
+    TLS is the Gateway's wildcard *.home.keithjackson.dev cert (no per-service cert step);
     ExternalDNS manages the record from the HTTPRoute (R-TLS/R-DNS, renderer-contract).
     """
     out = []

clusters/base/cluster-vars.yaml

@@ -26,49 +26,36 @@ data:
   # authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns
   # (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM.
   # See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN
-  # suffix, so the two clusters can no longer describe the same name.
+  # subdomain, so the two clusters can no longer describe the same name.
   #
-  #   dev   -> dev.home.keithjackson.dev        (a subdomain; = this base default)
-  #   prod  -> home.keithjackson.dev            (THE APEX — prod IS the apex, no `prod.` label)
+  #   dev   -> home.keithjackson.dev            (UNCHANGED — see below)
+  #   prod  -> prod.home.keithjackson.dev
   #
-  # OPERATOR DECISION 2026-09-04 (supersedes the interim split committed earlier the same day, which
-  # had prod on `prod.home.` with apex CNAMEs pointing at it). PROD IS THE APEX. That deletes the
-  # whole CNAME layer — `<svc>.home.keithjackson.dev` is prod's real name, not an alias — and it is
-  # DEV that carries a label. Convention is unchanged from GATEWAY_VIP / LONGHORN_DATA_PATH: the base
-  # default is the DEV value (dev restates it verbatim) and prod overrides it. What changed is which
-  # string dev holds.
-  #
-  # DEV'S VALUE IS A MIGRATION, NOT A VARIABLE CHANGE. The live dev cluster is serving the APEX right
-  # now: its wildcard cert, every public DNS record, its Keycloak issuer, ~12 OIDC clients' redirect
-  # URIs, kubectl SSO, and gitea.home.keithjackson.dev — which IS this repo's GitOps source AND the
-  # push target. Flipping this value on dev without first standing up the new host set STRANDS DEV:
-  # Flux's GitRepository url moves to gitea.dev.home.keithjackson.dev, which has no DNS record and no
-  # cert, so dev stops reconciling and cannot be fixed by pushing. ORDERED CUTOVER, operator present:
-  # infra/PROD-PROVISIONING.md §10.
-  #
-  # DEV MUST VACATE THE APEX BEFORE PROD PUBLISHES THERE. Both clusters describing
-  # `*.home.keithjackson.dev` at once is the 2026-09-04 incident precondition, re-created.
-  BASE_DOMAIN: "dev.home.keithjackson.dev"
+  # DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on
+  # `dev.home.keithjackson.dev` with apex CNAMEs `<svc>.home` -> `<svc>.prod.home`, but moving dev
+  # relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and
+  # the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2
+  # (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as
+  # GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates
+  # it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre-
+  # tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths).
+  BASE_DOMAIN: "home.keithjackson.dev"
 
   # ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind
   # EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered,
   # so it cannot be created, updated or deleted regardless of TXT ownership.
   #
-  # BOTH ENVIRONMENTS USE THE CLOUDFLARE **ZONE**, `keithjackson.dev`. This value is NOT a per-
-  # environment isolation knob and must not be narrowed to either environment's BASE_DOMAIN.
+  # DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev`
+  # apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter
+  # that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching
+  # record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged
+  # from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically.
   #
-  # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17) — the Cloudflare provider uses --domain-filter for ZONE
-  # SELECTION as well as name filtering, so a filter that is a STRICT SUBDOMAIN of the hosted zone
-  # matches NO zone: ExternalDNS generates the record and then skips it with "no hosted zone matching
-  # record DNS Name", publishing NOTHING. This has been tried and reverted more than once (an agent
-  # narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in review). DO NOT "harden"
-  # it — the change is not defence in depth, it is a total outage of DNS publication.
-  #
-  # WHERE ISOLATION ACTUALLY COMES FROM, and it is sufficient:
-  #   (1) the two environments request DISJOINT hostname sets (dev.home.… vs the apex), and
-  #   (2) EXTERNALDNS_TXT_OWNER_ID differs, so neither registry ever claims the other's records.
-  # That pairing is what the 2026-09-04 incident needed; a domain filter cannot provide it, because
-  # both environments live in the one zone either way.
+  # PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE
+  # (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned
+  # chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED
+  # — it can never delete another environment's records — and the fallback is to set prod's value
+  # back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard.
   EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev"
 
   # ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT:
@@ -77,35 +64,14 @@ data:
   # `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on
   # one zone therefore garbage-collect each other. Each environment MUST have a distinct value.
   #
-  #   dev -> `dev`      prod -> `prod`      base default -> a non-colliding sentinel (below)
+  # DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare
+  # under owner `default`; changing dev's value would make dev's own external-dns stop recognising
+  # them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it.
   #
-  # DEV IS `dev`, NOT `default` (changed 2026-09-04 with the move to dev.home.keithjackson.dev). Until
-  # this migration dev HAD to render `default`, because its LIVE apex records were already registered
-  # in Cloudflare's TXT registry under that owner and renaming would have orphaned every one of them.
-  # THAT CONSTRAINT NO LONGER BINDS: dev is publishing an ENTIRELY NEW record set under
-  # *.dev.home.keithjackson.dev, with no pre-existing registry entries to recognise, so the owner id
-  # is now free to be chosen on its merits. Two reasons to take `dev`:
-  #   - `default` is THE VALUE YOU GET BY FORGETTING (it is the external-dns chart default, and both
-  #     clusters silently shared it, which IS the incident). An environment whose identity is the
-  #     forget-value can never be told apart from a misconfigured newcomer. No environment should
-  #     hold it.
-  #   - It makes vacating the apex an EXPLICIT, ORDERED, VERIFIABLE step. Dev's old apex records stay
-  #     owned by `default`, which no environment now claims, so nothing deletes them implicitly on
-  #     some future reconcile — they are retired deliberately, and prod's publish is gated on that
-  #     retirement being confirmed in Cloudflare (infra/PROD-PROVISIONING.md §10) rather than on a
-  #     race between two reconcile loops.
-  #
-  # PROD STAYS `prod` AND MUST NOT BECOME `default`, even though prod now serves the apex names that
-  # dev's `default`-owned records currently occupy. Giving prod `default` would hand prod ownership of
-  # the LIVE dev cluster's records while dev is still serving them, and under `policy: sync` prod
-  # would delete the ones no prod source claims. That is the 2026-09-04 incident, re-armed, with the
-  # roles swapped. Prod inherits the apex NAMES; it must never inherit the apex OWNER ID.
-  #
-  # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default` and not any real
-  # environment's id. A new environment that forgets this knob must not silently inherit another's
-  # identity — that is precisely the incident. With this sentinel such a cluster owns nothing, so it
-  # can DELETE nothing; it would publish under a visibly wrong owner id (grep the TXT records)
-  # instead of destroying a live environment's DNS.
+  # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that
+  # forgets this knob must not silently inherit dev's identity — that is precisely the incident. With
+  # this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a
+  # visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS.
   EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars"
 
   ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
@@ -113,7 +79,7 @@ data:
                                             # ingress VIP of the `shared` Gateway. MUST equal the FIRST address of
                                             # ${CILIUM_LB_POOL_START}..${CILIUM_LB_POOL_STOP} below — the Gateway
                                             # takes the pool's first free IP, and coredns-custom answers
-                                            # *.${BASE_DOMAIN} with THIS so pod -> public-hostname traffic
+                                            # *.home.keithjackson.dev with THIS so pod -> public-hostname traffic
                                             # hairpins through the Gateway (correct SNI -> wildcard cert).
                                             # Base default = the DEV LAN VIP (was 10.73.0.240, an address on the
                                             # long-RETIRED libvirt network — corrected 2026-08-31 so an overlay that

clusters/dev/kustomization.yaml

@@ -5,20 +5,6 @@
 # reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02).
 # It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git
 # on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1).
-#
-# ##############################################################################################
-# ###  *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) ***                 ###
-# ###                                                                                        ###
-# ###  This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex   ###
-# ###  `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url       ###
-# ###  moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a   ###
-# ###  merge to main IS the cutover — it executes on the live dev cluster with nobody present. ###
-# ###  Dev's Gitea is both its GitOps source and the push target: if the new host is not up    ###
-# ###  with a Ready cert first, dev stops reconciling and pushing cannot repair it.            ###
-# ###                                                                                        ###
-# ###  Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator        ###
-# ###  present. Merging is step D3, not step one.                                             ###
-# ##############################################################################################
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
@@ -145,24 +131,7 @@ patches:
         # via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO
         # secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and
         # revert this url (docs/05).
-        #
-        # *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. ***
-        # It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized —
-        # it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO
-        # postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and
-        # break the source outright (the L0 flux-postbuild-substitution check asserts exactly this).
-        # So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below.
-        #
-        # It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That
-        # makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization
-        # (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE
-        # DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already
-        # resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED
-        # BY PUSHING — the broken thing is the path Flux uses to read pushes.
-        # ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root
-        # Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for
-        # the migration window, and only restore the in-cluster url after the new host is proven.
-        url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
+        url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git
         ref:
           branch: main
   # --- dev-scoped cluster-vars (R-ENV-02) ---
@@ -179,60 +148,33 @@ patches:
       - op: replace
         path: /data/ACME_CLUSTER_ISSUER
         value: "letsencrypt-prod"
-      # ==========================================================================================
-      # DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.**
-      # ==========================================================================================
-      # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day:
-      #
-      #   dev   -> dev.home.keithjackson.dev      (this knob; = the base default, restated)
-      #   prod  -> home.keithjackson.dev          (THE APEX — prod IS the apex, no `prod.` label,
-      #                                            and therefore no apex CNAME layer at all)
-      #
-      # *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right
-      # now. Merging this file does not perform the migration — it DEFINES the target. The cutover is
-      # an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10.
-      #
-      # WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20
-      # lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps
-      # source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute,
-      # DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does
-      # not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED
-      # BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery
-      # is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair
-      # in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host
-      # FIRST, or park the source on the bootstrap Gitea for the window.
-      #
-      # ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh
-      # ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s
-      # server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's
-      # split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex.
+      # --- DNS identity (R-DNS-01/02/03) -----------------------------------------------------
+      # DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly
+      # like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running.
+      # Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its
+      # Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev
+      # (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to
+      # prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did
+      # not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration,
+      # not a variable change: infra/PROD-PROVISIONING.md §10.
       - op: replace
         path: /data/BASE_DOMAIN
-        value: "dev.home.keithjackson.dev"
-      # ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING
-      # F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a
-      # strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching
-      # record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it
-      # newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held.
+        value: "home.keithjackson.dev"
+      # ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself,
+      # and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped
+      # the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held.
       - op: replace
         path: /data/EXTERNALDNS_DOMAIN_FILTER
         value: "keithjackson.dev"
-      # OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex
-      # records were already in Cloudflare's TXT registry under owner `default`, so renaming would
-      # have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix
-      # dev publishes a brand-new record set with no prior registry entries to recognise, so the id
-      # is chosen on its merits, and `default` has none — it is the value external-dns uses when you
-      # FORGET the flag, which is exactly how both clusters came to share it. Consequences, both
-      # deliberate:
-      #   - dev's OLD apex records remain owned by `default`, which no environment now claims, so
-      #     nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first
-      #     publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing.
-      #   - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev
-      #     cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER.
-      # dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel.
+      # DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT
+      # registry under owner `default` (the external-dns chart default, which is what both clusters
+      # silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records
+      # BY this string: any other value and it stops recognising every record it manages, then
+      # re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the
+      # whole point. The base default is a non-colliding sentinel, so this line is load-bearing.
       - op: replace
         path: /data/EXTERNALDNS_TXT_OWNER_ID
-        value: "dev"
+        value: "default"
       # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) -----------------------------
       # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it
       # equals the base default. These values are what the LIVE dev cluster is running right now, so
@@ -241,7 +183,7 @@ patches:
       #
       # Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN
       # (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers
-      # *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
+      # *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway.
       - op: replace
         path: /data/GATEWAY_VIP
         value: "192.168.1.53"

clusters/prod/kustomization.yaml

@@ -129,23 +129,14 @@ patches:
   #
   # WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the
   # unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`).
-  # REASONING UPDATED 2026-09-04 for the new suffix direction (prod = the apex, dev = dev.home.),
-  # which changes WHY the blocker holds but not THAT it holds:
-  #   1. SELF-HOSTING CIRCULARITY — still true, and now the only reason. The Gitea this repo is
-  #      pushed to runs ON THE DEV CLUSTER. Prod syncing from it would put prod's GitOps source
-  #      inside the very cluster slated for decommission. Gitea must move to prod (or off-cluster)
-  #      BEFORE dev is retired, or prod loses its source of truth.
-  #      NOTE the suffix change makes the two Giteas DIFFERENT HOSTS: dev's is now
-  #      gitea.dev.home.keithjackson.dev and prod's is gitea.home.keithjackson.dev. Prod pointing at
-  #      the apex name therefore means "prod's OWN Gitea", never dev's — which is what makes the
-  #      cutover below a one-line change once prod actually runs Gitea.
-  #   2. SPLIT-HORIZON DNS — NO LONGER A BLOCKER, and the note is kept because the old reasoning is
-  #      easy to re-derive incorrectly. platform/coredns/coredns-custom.yaml answers the whole
-  #      ${BASE_DOMAIN} zone with ${GATEWAY_VIP}; when prod's BASE_DOMAIN was `prod.home.` that meant
-  #      prod resolved gitea.home.keithjackson.dev to PROD's gateway (192.168.2.20) where no Gitea
-  #      listened — a misdirection. Now that prod IS the apex, resolving gitea.home.keithjackson.dev
-  #      to prod's own Gateway is CORRECT: that is where prod's Gitea will be served from. What
-  #      remains is purely reason 1 — the Gitea is not there YET.
+  # Two independent reasons, both consequences of the split-horizon design:
+  #   1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's
+  #      GitOps source lives inside the very cluster slated for decommission. Gitea must move to
+  #      prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth.
+  #   2. SPLIT-HORIZON DNS SENDS IT TO THE WRONG PLACE. platform/coredns/coredns-custom.yaml
+  #      answers the ENTIRE home.keithjackson.dev zone with ${GATEWAY_VIP}. On prod that is
+  #      192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve
+  #      gitea.home.keithjackson.dev to itself and never reach a repository.
   #
   # WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction):
   #   bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the
@@ -211,41 +202,35 @@ patches:
         path: /data/ACME_CLUSTER_ISSUER
         value: "letsencrypt-prod"
       # ==========================================================================================
-      # PROD DNS IDENTITY (R-DNS-01/02/03) — PROD **IS** THE APEX.
+      # PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident.
       # ==========================================================================================
-      # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day
-      # (prod on `prod.home.` behind apex CNAMEs). The simpler target, and the one authored here:
-      #
-      #   dev   -> dev.home.keithjackson.dev     (dev carries the label; clusters/dev)
-      #   prod  -> home.keithjackson.dev         (THE APEX — this knob)
-      #
-      # There is NO `prod.` label and NO CNAME layer. `<svc>.home.keithjackson.dev` is prod's REAL
-      # name, so nothing has to alias anything: prod's Gitea is gitea.home.keithjackson.dev, its
-      # Keycloak keycloak.home.keithjackson.dev, and the wildcard cert is `*.home.keithjackson.dev`.
-      #
-      # BACKGROUND — WHY THE SUFFIXES HAD TO SPLIT AT ALL. Until 2026-09-04 every hostname was
-      # HARDCODED to home.keithjackson.dev in the base manifests, so prod was authored to serve the
-      # EXACT SAME public names as the live dev cluster out of the SAME Cloudflare zone. Two
-      # consequences, one of which already happened:
+      # PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to
+      # home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME
+      # public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one
+      # of which already happened:
       #   1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit
       #      default dev had) concluded it owned dev's records and DELETED EVERY ONE of them.
       #   2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev`
       #      wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would
       #      have raced each other's `_acme-challenge` TXT records on that one zone.
+      # Moving prod to its own suffix dissolves both: the two clusters can no longer name the same
+      # record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here.
       #
-      # *** THE ORDERING CONSTRAINT THIS VALUE CREATES — READ BEFORE UNSUSPENDING ANYTHING. ***
-      # Prod now claims the very names the LIVE dev cluster is serving today. Both hazards above are
-      # therefore still armed until DEV HAS VACATED THE APEX. Prod's `platform` and its 10 downstream
-      # tiers are SUSPENDED for exactly this reason. The gate is not "review is done", it is:
-      #   (a) dev is reconciling on *.dev.home.keithjackson.dev with a Ready wildcard cert, AND
-      #   (b) dev's old apex A/CNAME records and their TXT registry entries are GONE from Cloudflare
-      #       (verified against Cloudflare directly — the workstation's dnsmasq is NOT evidence about public
-      #       DNS; that mistake was made on 2026-09-04), AND
-      #   (c) dev's old `*.home.keithjackson.dev` cert/Certificate is released, because prod ordering
-      #       that same name set while dev's still exists counts against LE's duplicate-certificate
-      #       limit (5 per week per identical name set).
-      # Full ordered runbook: infra/PROD-PROVISIONING.md §10. Unsuspending before (a)-(c) re-creates
-      # the 2026-09-04 incident with the roles reversed.
+      #   dev   -> home.keithjackson.dev         (UNCHANGED — dev is live; phase 2 moves it)
+      #   prod  -> prod.home.keithjackson.dev    (this knob)
+      #
+      # CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is
+      # gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so
+      # on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make
+      # `<svc>.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they
+      # would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10.
+      #
+      # THE APEX INVERSION IS DELIBERATELY NOT ON THIS BRANCH. Moving prod to the bare apex (and dev
+      # to dev.home.) is a LIVE MIGRATION of the dev cluster, not a variable change — see
+      # infra/PROD-PROVISIONING.md §10a. It is kept as a single commit on branch
+      # `migration/dns-apex-inversion` (9137bbd) and is applied as step D3 of that runbook, with the
+      # operator present. Everything else on this branch is independent of it, which is the whole
+      # reason it was split out.
       #
       # [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml.
       # The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is
@@ -253,33 +238,30 @@ patches:
       # trusts a different Keycloak than the one prod actually runs.
       - op: replace
         path: /data/BASE_DOMAIN
-        value: "home.keithjackson.dev"
+        value: "prod.home.keithjackson.dev"
+      # SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's
+      # OWN subtree, so prod cannot create, update or delete any record outside it no matter what the
+      # TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for
+      # ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped
+      # the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no
+      # matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's
+      # records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on
+      # EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own.
       - op: replace
         path: /data/EXTERNALDNS_DOMAIN_FILTER
-        # MUST be the Cloudflare ZONE. Prod now serves the apex, so `home.keithjackson.dev` looks
-        # like a tighter, more honest filter than the zone — IT IS NOT, IT IS AN OUTAGE. FINDING
-        # F-DNS1 (DECISIONS-LOG 2026-08-17) proved a STRICT SUBDOMAIN of the hosted zone drops the
-        # zone entirely: ExternalDNS generates the record then skips it with "no hosted zone matching
-        # record DNS Name", so prod publishes NOTHING. An agent narrowed this to
-        # prod.home.keithjackson.dev on 2026-09-04 and it was reverted in review; do not retry it.
-        # Isolation from dev does NOT come from this filter — it comes from
-        # (1) the two clusters requesting DISJOINT hostname sets (apex vs dev.home.), and
-        # (2) txtOwnerId: prod vs dev, so neither registry ever claims the other's records.
+        # MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG,
+        # 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the
+        # record then skipped it with "no hosted zone matching record DNS Name", so prod would
+        # publish NOTHING. Isolation from dev does NOT come from this filter — it comes from
+        # (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and
+        # (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records.
         # That pairing is what the 2026-09-04 incident needed and this filter cannot provide.
         value: "keithjackson.dev"
       # SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so
       # both inherited the chart default `default`; prod then treated dev's TXT-registry entries as
-      # its own and, under policy: sync, deleted the records behind them.
-      #
-      # PROD MUST NOT BE GIVEN `default`, however tempting it now looks. Prod is taking over the apex
-      # NAMES, and dev's existing apex records are registered under owner `default` — so `default` is
-      # the one value that would let prod adopt them. It would also let prod SYNC-DELETE the ones no
-      # prod source claims, on a cluster dev is still serving. Prod inherits the apex names; it must
-      # never inherit the apex OWNER ID. `prod` keeps prod fail-safe: records owned by `default` are
-      # invisible to it, so an early unsuspend cannot destroy dev's DNS the way 2026-09-04 did — it
-      # simply fails to publish until those records are retired, which is the correct behaviour.
-      # dev is `dev` (clusters/dev, changed with dev's migration); the base default is a
-      # non-colliding sentinel. All three differ.
+      # its own and, under policy: sync, deleted the records behind them. This value MUST differ from
+      # dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and
+      # cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix.
       - op: replace
         path: /data/EXTERNALDNS_TXT_OWNER_ID
         value: "prod"

infra/PROD-PROVISIONING.md

@@ -1177,7 +1177,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be
 | `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address |
 | `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address |
 | `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target |
-| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real. Prod's value was inverted the same day to the APEX `home.keithjackson.dev`, with dev moving to `dev.home.keithjackson.dev` — see §10a, which is authored but NOT yet applied. |
+| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. |
 
 - [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both*
       overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the
@@ -1331,12 +1331,10 @@ single-server-etcd window, and avoids a later member-add dance.
       specifically.
 - [ ] Monitoring and alerts live on prod.
 - [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03).
-      Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10a
-      "The L2 tests hardcode the apex".
-- [ ] **Prod serves the apex DNS/TLS identity** and dev's records are provably untouched (§10/§10a):
-      distinct `EXTERNALDNS_TXT_OWNER_ID` (`prod` vs `dev`), prod wildcard cert
-      `*.home.keithjackson.dev`, dev migrated to `*.dev.home.keithjackson.dev` with its old apex
-      records retired (§10a step D8), and prod's own Keycloak realm + per-service client secrets.
+      Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2.
+- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10):
+      distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and
+      prod's own Keycloak realm + per-service client secrets.
 
 ## 9a. PROD BRING-UP STATE (2026-09-04) — what came up, and the TWO blockers that need the operator
 
@@ -1435,6 +1433,18 @@ unsuspending `platform` and re-queried after — **57 records, byte-for-byte ide
 entries still `owner=default`.** Snapshots are taken against the **Cloudflare API**, never a local
 resolver (a pfSense override produced a false healthy reading during the incident).
 
+> ### THE APEX INVERSION IS NOT ON THIS BRANCH
+>
+> Sections 10 and 10a below describe the **target** DNS layout — prod on the bare apex, dev on
+> `dev.home.` — and the ordered runbook for getting there. **That change is deliberately not authored
+> on this branch.** It is one commit, `9137bbd`, kept on branch **`migration/dns-apex-inversion`**,
+> because merging it *is* dev's migration: `clusters/dev/sync.yaml` reconciles `./clusters/dev` from
+> `main` every 10 minutes, so a merge executes the cutover on the live dev cluster unattended.
+>
+> On this branch **prod is authored on `prod.home.keithjackson.dev`**, which is what prod actually
+> runs. Read what follows as the plan, not as a description of the current authored state. The
+> inversion is applied as step **D3** of the §10a runbook, operator present.
+
 ## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture**
 
 > **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's
@@ -1473,82 +1483,43 @@ values; for DNS and TLS that promise was false, so `clusters/prod` was *authored
 public identity. Two clusters that describe the same DNS name will fight over it no matter how the
 DNS controller is configured.
 
-### The architecture decision (operator) — **INVERTED 2026-09-04, this is the current target**
+### The architecture decision (operator)
 
 ```
-dev   ->  *.dev.home.keithjackson.dev      gitea.dev.home.keithjackson.dev
-prod  ->  *.home.keithjackson.dev          gitea.home.keithjackson.dev     (THE APEX)
+dev   ->  *.dev.home.keithjackson.dev        gitea.dev.home.keithjackson.dev
+prod  ->  *.prod.home.keithjackson.dev       gitea.prod.home.keithjackson.dev
+apex  ->  CNAME <svc>.home.keithjackson.dev  ->  <svc>.prod.home.keithjackson.dev
 ```
 
-**Prod *is* the apex. There is no `prod.` label and no CNAME layer at all** —
-`<svc>.home.keithjackson.dev` is prod's real name, not an alias for one. This **supersedes** the
-interim split committed earlier the same day (`c8c1d29`), which put prod on
-`*.prod.home.keithjackson.dev` behind apex CNAMEs pointing at it. That layer is deleted: fewer
-moving parts, no CNAME chain to keep in step, and prod ends up on the names users already type.
+### PHASE 1 — **prod only. Dev did not move.** (this change)
 
-The cost of the inversion is that **dev must move**, and dev is the live cluster. The interim split
-was chosen precisely because it moved only the unbuilt environment; this one does not have that
-property, so the whole of §10's remaining risk now sits on dev's side, and it is sequenced below.
+`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it.
 
-### The values, per environment
+| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` |
+|---|---|---|---|
+| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` |
+| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` |
+| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` |
 
-| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | Ansible `k3s_base_domain` |
-|---|---|---|---|---|
-| base default | `dev.home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | `dev.home.keithjackson.dev` (role default) |
-| **dev** | `dev.home.keithjackson.dev` | `dev` | `keithjackson.dev` | (inherits the role default) |
-| **prod** | `home.keithjackson.dev` | `prod` | `keithjackson.dev` | `home.keithjackson.dev` (`group_vars/k3s_prod`) |
+**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod
+alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken
+while the operator is away. Dev's rendered output is **byte-identical** except for one added line,
+`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified
+by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for
+both overlays and diffing.
 
-`[parity]`: `k3s_base_domain` must equal the same cluster's `BASE_DOMAIN`. The kube-apiserver's
-`oidc-issuer-url` is a k3s **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux
-post-build substitution can never reach it — hence one fact in two places. If they disagree the
-apiserver validates tokens against a different Keycloak than the cluster runs: kubectl SSO fails
-while the cert-based admin kubeconfig keeps working, so it presents as "SSO is broken", not as skew.
+**Two independent guards now, either sufficient alone:**
 
-**`EXTERNALDNS_DOMAIN_FILTER` is the Cloudflare ZONE for BOTH, and must stay that way.** FINDING
-F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for **zone selection** as
-well as record matching, so a filter that is a strict subdomain of the hosted zone matches **no
-zone** — external-dns generates the record and then skips it with *"no hosted zone matching record
-DNS Name"*, publishing **nothing**. This has been "hardened" and reverted more than once. It is not
-defence in depth; it is a total outage of DNS publication. Isolation comes from the two guards below,
-which a domain filter cannot provide anyway: both environments live in the one zone regardless.
+1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is
+   the direct fix for the incident. Dev **must** render exactly `default`: its live records are
+   already registered under that owner, and any other value would make dev's own external-dns stop
+   recognising them and re-create duplicates beside the orphans.
+2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree.
+   Defence in depth, added because the first guard is a single string in a single file.
 
-**The two guards that actually work:**
-
-1. **`txtOwnerId`, per-environment identity** — a cluster only ever reconciles records its own owner
-   id claims in the TXT registry. This is the direct fix for the incident.
-2. **Disjoint hostname sets** — after the migration the two clusters never describe the same record,
-   so there is nothing to fight over. *This guard does not exist during the migration window*, which
-   is the entire reason the ordering below is mandatory.
-
-#### Why dev's owner id changed from `default` to `dev`
-
-Until this migration dev **had** to render `default`: its live apex records were already registered
-in Cloudflare's TXT registry under that owner, and renaming would have orphaned every one of them.
-**The migration dissolves that constraint** — on `*.dev.home.keithjackson.dev` dev publishes an
-entirely new record set with no prior registry entries to recognise, so the id is free and is chosen
-on its merits. `default` has none: it is the value external-dns uses when you **forget** the flag,
-which is exactly how both clusters came to share it, and an environment whose identity is the
-forget-value can never be told apart from a misconfigured newcomer. Consequences, both deliberate:
-
-- Dev's **old apex records stay owned by `default`**, which no environment now claims, so nothing
-  deletes them implicitly on some future reconcile. Retiring them is an explicit step (**D7**) and
-  prod's first publish is gated on that retirement being *confirmed* — not on two reconcile loops
-  racing each other.
-- **Prod stays `prod` and must never be given `default`.** Prod is taking over the apex *names*, and
-  dev's existing apex records are registered under `default` — so `default` is the one value that
-  would let prod adopt them, and under `policy: sync` delete the ones no prod source claims, on a
-  cluster dev is still serving. That is the 2026-09-04 incident re-armed with the roles swapped.
-  Prod inherits the apex **names**; it must never inherit the apex **owner id**.
-
-All three values differ, and the base default remains a sentinel so an environment that forgets the
-knob owns nothing and can therefore delete nothing.
-
-### PHASE 1 — prod tokenized and moved off dev's names (`c8c1d29`, done; suffix since inverted)
-
-`BASE_DOMAIN` was a **dead knob** — declared in `clusters/base/cluster-vars.yaml` and referenced by
-no reconciled manifest, because every hostname was a hardcoded literal. `c8c1d29` made it real (141
-references across the reconciled set) and gave prod its own suffix. That work stands; only the
-*value* prod holds has changed, from `prod.home.keithjackson.dev` to the apex.
+The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a
+future environment that forgets the knob must not silently inherit dev's identity — that is exactly
+what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing.
 
 ### What could NOT be tokenized, and why
 
@@ -1557,49 +1528,49 @@ references across the reconciled set) and gave prod its own suffix. That work st
   Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and
   **cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake
   would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately.
-  **Both environments now need action:** dev's copy must move to `keycloak.dev.home.keithjackson.dev`
-  as part of step **D5**, and prod needs its own copy with `keycloak.home.keithjackson.dev` and a
-  client secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client
-  secret is realm-scoped.
+  **Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak.
+  Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client
+  secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret
+  is per-realm.
 - **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s
-  server flag, not a manifest. Handled on the **Ansible** seam instead: role default
-  `k3s_base_domain` (dev's value) + derived `k3s_oidc_issuer_url`, overridden for prod in
-  `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`,
-  so a fourth prod server inherits it from group membership. Verified per host by a read-only probe
-  (`connection: local`, zero SSH, `changed=0`): all three dev servers resolve
-  `https://keycloak.dev.home.keithjackson.dev/realms/homelab`, all three prod servers resolve
-  `https://keycloak.home.keithjackson.dev/realms/homelab`.
-- **`spec.url` of dev's `GitRepository`** (`clusters/dev/kustomization.yaml`) — see the strand below.
-  It *cannot* be tokenized: the `flux-system` root Kustomization that renders `./clusters/dev`
-  declares no `postBuild.substituteFrom`, so `gitea.${BASE_DOMAIN}` there would ship the **literal
-  token** and break the source outright (the L0 `flux-postbuild-substitution` check asserts exactly
-  this). It is a hand-maintained literal that must move in lockstep with `BASE_DOMAIN`.
+  **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution
+  can never reach it. Correctly tokenized on the **Ansible** side instead: new role default
+  `k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in
+  `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`.
+  `[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver
+  validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the
+  cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew).
 
----
+### A pre-existing prod problem this incidentally FIXED
 
-## 10a. THE MIGRATION RUNBOOK — ordering is load-bearing
+§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC
+issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN,
+making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on
+its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by
+**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer
+and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's
+external-dns has actually published the record; until then use the break-glass cert kubeconfig.
 
-> **Prod's `platform` tier and its 10 downstream tiers are SUSPENDED and must stay suspended until
-> step P1.** Nothing below is safe to run out of order, and nothing below should be run while the
-> operator is away.
+### Verify on prod's first reconcile
 
-### The two ways this goes wrong
+- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup
+      as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone
+      ("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it
+      will create **nothing** — fail-closed, never destructive. Fallback: set prod's
+      `EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is
+      sufficient on its own.
+- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching
+      dev's `_acme-challenge` records.
+- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry
+      entries still read owner `default` and that prod's read owner `prod`.
+- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client
+      secret is realm-scoped; prod cannot reuse dev's).
 
-**(1) BOTH CLUSTERS ON THE APEX — the 2026-09-04 incident, recreated.** If prod publishes
-`*.home.keithjackson.dev` before dev has vacated it, two clusters describe the same records in the
-same zone again. Distinct `txtOwnerId`s stop prod from *deleting* dev's records — that guard holds —
-but they do not stop the second failure mode: both clusters requesting a `letsencrypt-prod` wildcard
-for the **same name set** and racing each other's `_acme-challenge` TXT records, so neither reliably
-solves DNS-01. **Rule: dev vacates the apex first, and the vacate is verified against Cloudflare
-before prod is unsuspended.**
+### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**)
 
-**(2) THE STRAND — dev's GitOps source is the thing that moves.** `spec.url` of dev's
-`GitRepository` is `https://gitea.home.keithjackson.dev/...`, which is **both** dev's GitOps source
-**and** the PM's push target. Flipping dev's `BASE_DOMAIN` moves Gitea's HTTPRoute, its public DNS
-record and the wildcard cert covering it to `gitea.dev.home.keithjackson.dev` all at once. If that
-host does not already resolve and serve a valid, Ready cert at that instant, **dev stops reconciling
-and cannot be fixed by pushing** — the broken thing *is* the path Flux uses to read pushes. Recovery
-is then hands-on and needs LAN access to the cluster.
+Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the
+Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change.
+Required, roughly in order:
 
 **(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root
 Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main`

infra/ansible/inventory/group_vars/k3s_prod/vars.yml

@@ -306,47 +306,35 @@ node_baseline_dns_servers:
   # k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through
   # these PUBLIC upstreams — not through CoreDNS's split-horizon answer.
   #
-  # ORDERING HAZARD, NOT YET CLEARED. Prod's issuer is keycloak.home.keithjackson.dev — an APEX
-  # name, because prod IS the apex under the 2026-09-04 inversion (k3s_base_domain below).
-  # That is the name the LIVE DEV CLUSTER answers today, and its public A record currently
-  # points at DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. A prod
-  # node resolving its own issuer through these public upstreams therefore reaches dev until:
-  #   (1) dev has migrated to *.dev.home.keithjackson.dev and RETIRED its apex records, and
-  #   (2) PROD's own external-dns has published keycloak.home.keithjackson.dev at PROD's
-  #       Gateway VIP (192.168.2.20).
-  # Only then do the public answer and the in-cluster split-horizon answer AGREE, and no
-  # /etc/hosts pin is needed. Until then kubectl OIDC SSO on prod cannot work; the cert-based
-  # admin kubeconfig is the break-glass path and always works. Ordering, not a design gap —
-  # the ordered steps are infra/PROD-PROVISIONING.md §10.
+  # THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was
+  # authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev
+  # publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so
+  # kubectl OIDC SSO on prod could not work at all. Now prod's issuer is
+  # keycloak.prod.home.keithjackson.dev, and the public record for that name is published by
+  # PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer
+  # and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed.
+  #
+  # STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's
+  # external-dns has actually published the record. Until then the issuer does not resolve
+  # from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass
+  # path, and it always works. Ordering, not a design gap.
 node_baseline_dns_search: []
 
 # --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) --------------------
-# PROD **IS** THE APEX. [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml —
-# the same one-fact-in-two-places contract as k3s_data_dir and k3s_api_host. The Flux side
-# cannot supply this: the kube-apiserver's oidc-issuer-url is a k3s server FLAG written to
-# /etc/rancher/k3s/config.yaml, not a manifest, so postBuild.substituteFrom never sees it.
-# Full rationale in roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url derives.
+# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/
+# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and
+# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a
+# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so
+# postBuild.substituteFrom never sees it. Full rationale in
+# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this.
 #
-# WHY THE SUFFIXES SPLIT (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve
-# the SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
-# deleted every one of dev's public records.
-#
-# WHICH WAY ROUND — INVERTED 2026-09-04 by operator decision, superseding the interim split
-# committed earlier the same day (prod on `prod.home.` behind apex CNAMEs). The simpler target:
-#   prod -> home.keithjackson.dev       (THE APEX; this variable — no `prod.` label, no CNAMEs)
-#   dev  -> dev.home.keithjackson.dev   (roles/k3s_server/defaults/main.yml)
-# So prod's Keycloak issuer is https://keycloak.home.keithjackson.dev/realms/homelab, and the
-# `kubernetes` OIDC client in PROD's OWN Keycloak realm is what prod's kubectl SSO authenticates
-# against.
-#
-# ORDERING HAZARD SPECIFIC TO THIS VARIABLE. keycloak.home.keithjackson.dev is the name the LIVE
-# dev cluster answers today, and prod's nodes resolve the issuer through PUBLIC upstreams (see
-# node_baseline_dns_search above). So until dev has vacated the apex and PROD's external-dns has
-# published that name at PROD's Gateway VIP (192.168.2.20), a prod node resolving its own issuer
-# gets DEV's VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. kubectl OIDC SSO on prod
-# therefore does not work until §10's dev-vacate steps are complete; the cert-based admin
-# kubeconfig is the break-glass path and always works. Ordering, not a design gap.
-k3s_base_domain: "home.keithjackson.dev"
+# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the
+# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns
+# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE);
+# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is
+# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client
+# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against.
+k3s_base_domain: "prod.home.keithjackson.dev"
 
 # --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) -----
 # roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE

infra/ansible/roles/k3s_server/defaults/main.yml

@@ -95,26 +95,15 @@ k3s_tls_sans: >-
 # while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken",
 # not like a config skew. The break-glass cert kubeconfig is why this is recoverable.
 #
-# THE DEFAULT IS THE DEV VALUE — the same convention as k3s_data_dir above and cluster-vars'
-# GATEWAY_VIP / LONGHORN_DATA_PATH: the role default is dev's value and prod overrides it in
-# inventory/group_vars/k3s_prod/vars.yml (the same inventory seam k3s_data_dir / k3s_api_host /
-# kube_vip_address use, so a FOURTH prod server inherits it from group membership rather than
-# from anyone remembering).
+# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same
+# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the
+# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte
+# what it already is and the Restart handler does NOT fire on a re-run.
 #
-# *** WHICH DEV VALUE — UPDATED 2026-09-04, AND IT IS AHEAD OF THE LIVE DEV CLUSTER. ***
-# The operator inverted the suffix split: PROD is now the apex `home.keithjackson.dev` and DEV
-# carries the label `dev.home.keithjackson.dev`. This default therefore moved from the apex to
-# dev's subdomain, mirroring BASE_DOMAIN in clusters/dev/kustomization.yaml.
-#
-# CONSEQUENCE — this default no longer matches what the live dev nodes are running, so unlike
-# every previous edit to this file A DEV RE-RUN IS NOT A NO-OP. Rendering config.yaml with the
-# new issuer CHANGES the file, FIRES the Restart handler, and rolls k3s on all three dev
-# servers, which also invalidates every user's OIDC kubeconfig (SSO must be re-issued). That is
-# a deliberate step of the dev DNS migration, NOT a side effect to discover during unrelated
-# work: DO NOT run the k3s_server role against dev until step D6 of
-# infra/PROD-PROVISIONING.md §10, and never while the operator is away. The break-glass
-# cert-based admin kubeconfig keeps working throughout, which is what makes the step safe.
-k3s_base_domain: "dev.home.keithjackson.dev"
+# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam
+# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it
+# from group membership rather than from anyone remembering.
+k3s_base_domain: "home.keithjackson.dev"
 
 # Derived, so the realm path is stated once. Override k3s_base_domain, not this.
 k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab"

platform/externaldns/helmrelease.yaml

@@ -39,33 +39,24 @@ spec:
     # garbage-collected them. `policy: sync` is correct and stays; what was missing was identity.
     # Full write-up: infra/PROD-PROVISIONING.md §10.
     #
-    # THE TWO GUARDS THAT ACTUALLY PREVENT A RECURRENCE — note that neither is domainFilters:
-    #   1. txtOwnerId  — per-environment identity; a cluster only ever touches records it owns.
-    #   2. DISJOINT HOSTNAME SETS — the two environments serve different suffixes, so they never
-    #      describe the same record in the first place. dev = *.dev.home.keithjackson.dev,
-    #      prod = *.home.keithjackson.dev (prod IS the apex; operator decision 2026-09-04).
+    # TWO INDEPENDENT GUARDS NOW, either of which alone prevents a recurrence:
+    #   1. txtOwnerId       — per-environment identity; a cluster only ever touches records it owns.
+    #   2. domainFilters    — per-environment name scope; prod cannot even see names outside its
+    #                         own subtree. Defence in depth, added after the incident.
     # ---------------------------------------------------------------------------------------------
-    # Which names this instance may manage at all. BOTH ENVIRONMENTS RENDER THE CLOUDFLARE **ZONE**
-    # `keithjackson.dev` HERE, and this is NOT an oversight to be tightened.
-    #
-    # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for ZONE
-    # SELECTION as well as record matching, so a filter that is a STRICT SUBDOMAIN of the hosted zone
-    # matches NO zone — external-dns generates the record and then skips it with "no hosted zone
-    # matching record DNS Name", publishing NOTHING. This has been "improved" and reverted more than
-    # once (an agent narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in
-    # review). It is not defence in depth; it is a total outage of DNS publication. Isolation comes
-    # from the two guards above, which a domain filter cannot provide anyway — both environments live
-    # in the ONE zone regardless of what this says. See clusters/base/cluster-vars.yaml.
+    # Which names this instance may manage at all. Cloudflare-provider caveat, empirically hit
+    # 2026-08: this value is used for ZONE selection as well as record matching, and a filter that is
+    # a strict subdomain of the zone dropped the parent zone entirely ("no hosted zone matching record
+    # DNS Name") — so DEV keeps the zone apex `keithjackson.dev`, which is also what dev serves the
+    # apex of. PROD narrows to `prod.home.keithjackson.dev`. If prod's first reconcile logs no
+    # matching zone, that caveat still applies: prod creates NOTHING (fail-closed, never destructive)
+    # and the fallback is to widen prod's EXTERNALDNS_DOMAIN_FILTER back to the zone apex, leaving
+    # txtOwnerId as the sole guard. See clusters/base/cluster-vars.yaml.
     domainFilters:
       - "${EXTERNALDNS_DOMAIN_FILTER}"
     # The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence
-    # caused the incident. dev renders `dev`, prod renders `prod`, and the BASE default is a
-    # non-colliding sentinel so a new environment that forgets the knob owns nothing and can delete
-    # nothing. NOTHING renders `default` any more (the external-dns chart default, and the value both
-    # clusters silently shared): dev held it only because its live apex records were registered under
-    # it, and dev's move to *.dev.home.keithjackson.dev retired that constraint along with those
-    # records. Prod must NOT be given `default` even though it inherits the apex NAMES — that would
-    # hand prod ownership of dev's old apex records and let `policy: sync` below delete them.
+    # caused the incident. Dev renders `default` (its live records are already registered under that
+    # owner; changing it would orphan them and duplicate every record). Prod renders `prod`.
     txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}"
     policy: sync            # create + update + delete records as services come and go — but ONLY
                             # records this owner id owns AND this domainFilter admits (see above).