diff --git a/catalog/render.py b/catalog/render.py index ef2c611..16b4323 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -47,8 +47,7 @@ LBL = "homelab.keithjackson.dev" # The platform hostname suffix, emitted as the Flux post-build TOKEN rather than a literal. # Every rendered manifest lands under a Flux path whose Kustomization declares # postBuild.substituteFrom cluster-vars, so `${BASE_DOMAIN}` is resolved per environment at -# reconcile time: dev renders dev.home.keithjackson.dev, prod renders home.keithjackson.dev (the -# APEX — operator decision 2026-09-04; prod IS the apex, so there is no `prod.` label and no CNAMEs). +# reconcile time: dev renders home.keithjackson.dev, prod renders prod.home.keithjackson.dev. # It is a DECLARED cluster-vars knob (clusters/base/cluster-vars.yaml), so it is written BARE — # the escape contract below reserves `$${...}` for expansions meant for a container's shell. # Hardcoding the literal here is what let prod be authored to serve dev's public hostnames, @@ -607,7 +606,7 @@ def r_internet_egress(s: dict) -> dict: def r_httproutes(s: dict) -> list: """One HTTPRoute per ingressRoute, attaching host -> service port on the shared Gateway. - TLS is the Gateway's wildcard *.${BASE_DOMAIN} cert (no per-service cert step); + TLS is the Gateway's wildcard *.home.keithjackson.dev cert (no per-service cert step); ExternalDNS manages the record from the HTTPRoute (R-TLS/R-DNS, renderer-contract). """ out = [] diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 8aa6544..b45426f 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -26,49 +26,36 @@ data: # authored to serve the SAME hostnames out of the SAME Cloudflare zone, so prod's external-dns # (policy: sync, owner `default`) concluded it owned dev's public records and DELETED ALL OF THEM. # See infra/PROD-PROVISIONING.md §10. The fix is architectural: each environment gets its OWN - # suffix, so the two clusters can no longer describe the same name. + # subdomain, so the two clusters can no longer describe the same name. # - # dev -> dev.home.keithjackson.dev (a subdomain; = this base default) - # prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label) + # dev -> home.keithjackson.dev (UNCHANGED — see below) + # prod -> prod.home.keithjackson.dev # - # OPERATOR DECISION 2026-09-04 (supersedes the interim split committed earlier the same day, which - # had prod on `prod.home.` with apex CNAMEs pointing at it). PROD IS THE APEX. That deletes the - # whole CNAME layer — `.home.keithjackson.dev` is prod's real name, not an alias — and it is - # DEV that carries a label. Convention is unchanged from GATEWAY_VIP / LONGHORN_DATA_PATH: the base - # default is the DEV value (dev restates it verbatim) and prod overrides it. What changed is which - # string dev holds. - # - # DEV'S VALUE IS A MIGRATION, NOT A VARIABLE CHANGE. The live dev cluster is serving the APEX right - # now: its wildcard cert, every public DNS record, its Keycloak issuer, ~12 OIDC clients' redirect - # URIs, kubectl SSO, and gitea.home.keithjackson.dev — which IS this repo's GitOps source AND the - # push target. Flipping this value on dev without first standing up the new host set STRANDS DEV: - # Flux's GitRepository url moves to gitea.dev.home.keithjackson.dev, which has no DNS record and no - # cert, so dev stops reconciling and cannot be fixed by pushing. ORDERED CUTOVER, operator present: - # infra/PROD-PROVISIONING.md §10. - # - # DEV MUST VACATE THE APEX BEFORE PROD PUBLISHES THERE. Both clusters describing - # `*.home.keithjackson.dev` at once is the 2026-09-04 incident precondition, re-created. - BASE_DOMAIN: "dev.home.keithjackson.dev" + # DEV DELIBERATELY STAYS ON THE APEX `home.keithjackson.dev`. The target architecture puts dev on + # `dev.home.keithjackson.dev` with apex CNAMEs `.home` -> `.prod.home`, but moving dev + # relocates its Keycloak issuer, every OIDC client redirect URI, kubectl SSO, its wildcard cert and + # the Gitea host that IS the GitOps source — on a live cluster. Deferred to phase 2 + # (infra/PROD-PROVISIONING.md §10). Base default is therefore the DEV value, the same convention as + # GATEWAY_VIP / LONGHORN_DATA_PATH: the base default is the known-good working value, dev restates + # it verbatim, and prod overrides it. Dev's rendered output is byte-identical to the pre- + # tokenization hardcoded manifests (verified 2026-09-04, all 12 substituted paths). + BASE_DOMAIN: "home.keithjackson.dev" # ExternalDNS `--domain-filter` (platform/externaldns/helmrelease.yaml). SAFETY NET #2 behind # EXTERNALDNS_TXT_OWNER_ID: a record whose name falls outside this filter is not even considered, # so it cannot be created, updated or deleted regardless of TXT ownership. # - # BOTH ENVIRONMENTS USE THE CLOUDFLARE **ZONE**, `keithjackson.dev`. This value is NOT a per- - # environment isolation knob and must not be narrowed to either environment's BASE_DOMAIN. + # DEV KEEPS THE ZONE APEX `keithjackson.dev`, and must: dev serves the `home.keithjackson.dev` + # apex itself, and the Cloudflare provider filters the ACCOUNT'S ZONES by this value — a filter + # that is a strict subdomain of the zone dropped the parent zone outright ("no hosted zone matching + # record DNS Name", DECISIONS-LOG 2026-08) on the version in use then. So dev's value is unchanged + # from the retired `DNS_ZONE` knob this replaces, and dev renders byte-identically. # - # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17) — the Cloudflare provider uses --domain-filter for ZONE - # SELECTION as well as name filtering, so a filter that is a STRICT SUBDOMAIN of the hosted zone - # matches NO zone: ExternalDNS generates the record and then skips it with "no hosted zone matching - # record DNS Name", publishing NOTHING. This has been tried and reverted more than once (an agent - # narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in review). DO NOT "harden" - # it — the change is not defence in depth, it is a total outage of DNS publication. - # - # WHERE ISOLATION ACTUALLY COMES FROM, and it is sufficient: - # (1) the two environments request DISJOINT hostname sets (dev.home.… vs the apex), and - # (2) EXTERNALDNS_TXT_OWNER_ID differs, so neither registry ever claims the other's records. - # That pairing is what the 2026-09-04 incident needed; a domain filter cannot provide it, because - # both environments live in the one zone either way. + # PROD NARROWS TO ITS OWN SUBTREE `prod.home.keithjackson.dev`. VERIFY ON PROD'S FIRST RECONCILE + # (infra/PROD-PROVISIONING.md §10): if the zone-lookup limitation above still applies to the pinned + # chart, prod's external-dns will select NO zone and create NO records. That failure is FAIL-CLOSED + # — it can never delete another environment's records — and the fallback is to set prod's value + # back to "keithjackson.dev", leaving EXTERNALDNS_TXT_OWNER_ID as the sole (and sufficient) guard. EXTERNALDNS_DOMAIN_FILTER: "keithjackson.dev" # ExternalDNS TXT registry owner id (`--txt-owner-id`). THE ROOT CAUSE OF THE 2026-09-04 INCIDENT: @@ -77,35 +64,14 @@ data: # `policy: sync`, to DELETE when no local resource claims it. Two clusters sharing an owner id on # one zone therefore garbage-collect each other. Each environment MUST have a distinct value. # - # dev -> `dev` prod -> `prod` base default -> a non-colliding sentinel (below) + # DEV MUST RENDER EXACTLY `default`. Dev's LIVE public records are already registered in Cloudflare + # under owner `default`; changing dev's value would make dev's own external-dns stop recognising + # them, and it would re-create duplicates alongside the orphaned originals. Dev's overlay pins it. # - # DEV IS `dev`, NOT `default` (changed 2026-09-04 with the move to dev.home.keithjackson.dev). Until - # this migration dev HAD to render `default`, because its LIVE apex records were already registered - # in Cloudflare's TXT registry under that owner and renaming would have orphaned every one of them. - # THAT CONSTRAINT NO LONGER BINDS: dev is publishing an ENTIRELY NEW record set under - # *.dev.home.keithjackson.dev, with no pre-existing registry entries to recognise, so the owner id - # is now free to be chosen on its merits. Two reasons to take `dev`: - # - `default` is THE VALUE YOU GET BY FORGETTING (it is the external-dns chart default, and both - # clusters silently shared it, which IS the incident). An environment whose identity is the - # forget-value can never be told apart from a misconfigured newcomer. No environment should - # hold it. - # - It makes vacating the apex an EXPLICIT, ORDERED, VERIFIABLE step. Dev's old apex records stay - # owned by `default`, which no environment now claims, so nothing deletes them implicitly on - # some future reconcile — they are retired deliberately, and prod's publish is gated on that - # retirement being confirmed in Cloudflare (infra/PROD-PROVISIONING.md §10) rather than on a - # race between two reconcile loops. - # - # PROD STAYS `prod` AND MUST NOT BECOME `default`, even though prod now serves the apex names that - # dev's `default`-owned records currently occupy. Giving prod `default` would hand prod ownership of - # the LIVE dev cluster's records while dev is still serving them, and under `policy: sync` prod - # would delete the ones no prod source claims. That is the 2026-09-04 incident, re-armed, with the - # roles swapped. Prod inherits the apex NAMES; it must never inherit the apex OWNER ID. - # - # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default` and not any real - # environment's id. A new environment that forgets this knob must not silently inherit another's - # identity — that is precisely the incident. With this sentinel such a cluster owns nothing, so it - # can DELETE nothing; it would publish under a visibly wrong owner id (grep the TXT records) - # instead of destroying a live environment's DNS. + # THE BASE DEFAULT IS A NON-COLLIDING SENTINEL, deliberately NOT `default`. A new environment that + # forgets this knob must not silently inherit dev's identity — that is precisely the incident. With + # this sentinel such a cluster owns nothing, so it can DELETE nothing; it would publish under a + # visibly wrong owner id (grep the TXT records) instead of destroying a live environment's DNS. EXTERNALDNS_TXT_OWNER_ID: "unset-see-cluster-vars" ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006) @@ -113,7 +79,7 @@ data: # ingress VIP of the `shared` Gateway. MUST equal the FIRST address of # ${CILIUM_LB_POOL_START}..${CILIUM_LB_POOL_STOP} below — the Gateway # takes the pool's first free IP, and coredns-custom answers - # *.${BASE_DOMAIN} with THIS so pod -> public-hostname traffic + # *.home.keithjackson.dev with THIS so pod -> public-hostname traffic # hairpins through the Gateway (correct SNI -> wildcard cert). # Base default = the DEV LAN VIP (was 10.73.0.240, an address on the # long-RETIRED libvirt network — corrected 2026-08-31 so an overlay that diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index e321d62..77aefc5 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -5,20 +5,6 @@ # reconciliation graph and patches the shared cluster-vars ConfigMap with dev-scoped values (R-ENV-02). # It also adds a self-managing ROOT Kustomization (sync.yaml) so the whole graph reconciles from git # on every push — no more manual `kubectl apply -k` for cluster-vars/overlay changes (F-OPS1). -# -# ############################################################################################## -# ### *** DO NOT MERGE THIS FILE TO `main` UNATTENDED (as of 2026-09-04) *** ### -# ### ### -# ### This overlay now carries dev's DNS MIGRATION target: BASE_DOMAIN moves from the apex ### -# ### `home.keithjackson.dev` to `dev.home.keithjackson.dev`, and the GitRepository url ### -# ### moves with it. sync.yaml reconciles ./clusters/dev from `main` EVERY 10 MINUTES, so a ### -# ### merge to main IS the cutover — it executes on the live dev cluster with nobody present. ### -# ### Dev's Gitea is both its GitOps source and the push target: if the new host is not up ### -# ### with a Ready cert first, dev stops reconciling and pushing cannot repair it. ### -# ### ### -# ### Run the ordered runbook: infra/PROD-PROVISIONING.md §10 (steps D1-D8), operator ### -# ### present. Merging is step D3, not step one. ### -# ############################################################################################## apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: @@ -145,24 +131,7 @@ patches: # via coredns -> Gateway VIP, and the LE-prod wildcard cert is publicly trusted so NO # secretRef/caCert is needed. Recovery: restart bootstrap Gitea (192.168.2.103:3000) and # revert this url (docs/05). - # - # *** THE STRAND. THIS URL IS THE MOST DANGEROUS LINE IN THE REPOSITORY. *** - # It is BOTH dev's GitOps source AND the operator's push target, and it is NOT tokenized — - # it cannot be: the `flux-system` root Kustomization that renders ./clusters/dev declares NO - # postBuild.substituteFrom, so `gitea.${BASE_DOMAIN}` here would ship the LITERAL token and - # break the source outright (the L0 flux-postbuild-substitution check asserts exactly this). - # So it is a hand-maintained literal that must be moved in lockstep with BASE_DOMAIN below. - # - # It has been moved to dev's NEW suffix as part of the 2026-09-04 migration authoring. That - # makes this file the TARGET state, NOT a safe unattended merge: dev's root Kustomization - # (sync.yaml) reconciles ./clusters/dev from `main` every 10m, so MERGING THIS TO main WHILE - # DEV IS LIVE MIGRATES DEV UNATTENDED. If gitea.dev.home.keithjackson.dev does not already - # resolve and serve a valid cert at that instant, dev stops reconciling and CANNOT BE FIXED - # BY PUSHING — the broken thing is the path Flux uses to read pushes. - # ORDERING THAT AVOIDS IT (infra/PROD-PROVISIONING.md §10, steps D1-D3): suspend dev's root - # Kustomization or park this url on the bootstrap Gitea (http://192.168.2.132:3000/...) for - # the migration window, and only restore the in-cluster url after the new host is proven. - url: https://gitea.dev.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git + url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git ref: branch: main # --- dev-scoped cluster-vars (R-ENV-02) --- @@ -179,60 +148,33 @@ patches: - op: replace path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" - # ========================================================================================== - # DEV DNS IDENTITY (R-DNS-01/02/03) — DEV MOVES OFF THE APEX. **NOT YET APPLIED.** - # ========================================================================================== - # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day: - # - # dev -> dev.home.keithjackson.dev (this knob; = the base default, restated) - # prod -> home.keithjackson.dev (THE APEX — prod IS the apex, no `prod.` label, - # and therefore no apex CNAME layer at all) - # - # *** THIS VALUE IS AHEAD OF THE LIVE CLUSTER. *** The dev cluster is serving the APEX right - # now. Merging this file does not perform the migration — it DEFINES the target. The cutover is - # an ORDERED, OPERATOR-PRESENT runbook: infra/PROD-PROVISIONING.md §10. - # - # WHY IT CANNOT BE A PLAIN VARIABLE FLIP — THE STRAND. `spec.url` of the GitRepository ~20 - # lines above is https://gitea.home.keithjackson.dev/... . Dev's Gitea is BOTH dev's GitOps - # source AND the operator's push target. The moment BASE_DOMAIN moves, dev's Gitea HTTPRoute, - # DNS record and wildcard cert all move to gitea.dev.home.keithjackson.dev. If that host does - # not ALREADY resolve with a valid, Ready cert, dev stops reconciling — and it CANNOT BE FIXED - # BY PUSHING, because the thing that is broken is the path Flux uses to read pushes. Recovery - # is then hands-on (point the source at the bootstrap Gitea on 192.168.2.132:3000, or repair - # in-cluster by hand). §10 states the ordering that avoids it: publish and verify the new host - # FIRST, or park the source on the bootstrap Gitea for the window. - # - # ALSO MOVED BY THIS ONE LINE, all on a live cluster: the `*.` wildcard Certificate (a fresh - # ACME order), Keycloak's issuer URL and ~12 OIDC clients' redirect URIs, kubectl SSO (a k3s - # server flag => a rolling restart + every user re-issuing their kubeconfig), coredns-custom's - # split-horizon zone, Homepage's links, and 6 L2 chainsaw tests that hardcode the apex. + # --- DNS identity (R-DNS-01/02/03) ----------------------------------------------------- + # DEV STAYS ON THE APEX. Restated explicitly even though it equals the base default, exactly + # like the LAN block and LONGHORN_DATA_PATH below: this is what the LIVE cluster is running. + # Every one of dev's public DNS records, its `*.home.keithjackson.dev` wildcard cert, its + # Keycloak issuer, every OIDC client redirect URI, kubectl SSO, and gitea.home.keithjackson.dev + # (which IS this repo's GitOps source) are all on this exact suffix RIGHT NOW. Prod moved to + # prod.home.keithjackson.dev instead (2026-09-04) — that alone ended the collision, so dev did + # not have to move. Relocating dev to dev.home.keithjackson.dev is phase 2 and is a migration, + # not a variable change: infra/PROD-PROVISIONING.md §10. - op: replace path: /data/BASE_DOMAIN - value: "dev.home.keithjackson.dev" - # ExternalDNS scope. MUST stay the Cloudflare ZONE — NOT dev.home.keithjackson.dev. FINDING - # F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this for ZONE SELECTION, so a - # strict-subdomain filter matches no zone and publishes NOTHING ("no hosted zone matching - # record DNS Name"). Dev moving to a subdomain does NOT make narrowing safe — it makes it - # newly tempting, which is why this comment is here. Same value the retired DNS_ZONE knob held. + value: "home.keithjackson.dev" + # ExternalDNS scope. MUST stay the zone apex: dev serves the home.keithjackson.dev APEX itself, + # and the Cloudflare provider selects zones with this value (a strict-subdomain filter dropped + # the parent zone — DECISIONS-LOG 2026-08). Same value the retired DNS_ZONE knob held. - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER value: "keithjackson.dev" - # OWNER ID: `dev` (was `default` until 2026-09-04). The old value was FORCED — dev's live apex - # records were already in Cloudflare's TXT registry under owner `default`, so renaming would - # have orphaned every one of them. THE MIGRATION DISSOLVES THAT CONSTRAINT: on the new suffix - # dev publishes a brand-new record set with no prior registry entries to recognise, so the id - # is chosen on its merits, and `default` has none — it is the value external-dns uses when you - # FORGET the flag, which is exactly how both clusters came to share it. Consequences, both - # deliberate: - # - dev's OLD apex records remain owned by `default`, which no environment now claims, so - # nothing deletes them implicitly. Retiring them is an explicit §10 step, and prod's first - # publish is gated on that retirement being CONFIRMED — not on two reconcile loops racing. - # - prod stays `prod` and must NEVER be given `default`, or prod would own the live dev - # cluster's apex records and sync-delete them. Prod inherits the apex NAMES, not its OWNER. - # dev != prod is the load-bearing invariant; the base default stays a non-colliding sentinel. + # DO NOT CHANGE THIS VALUE. Dev's LIVE public records are registered in Cloudflare's TXT + # registry under owner `default` (the external-dns chart default, which is what both clusters + # silently shared until the 2026-09-04 incident). Dev's external-dns identifies its own records + # BY this string: any other value and it stops recognising every record it manages, then + # re-creates duplicates beside the orphaned originals. Prod is `prod` — distinct, which is the + # whole point. The base default is a non-colliding sentinel, so this line is load-bearing. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID - value: "dev" + value: "default" # --- dev LAN addressing (192.168.1.0/24 on the Proxmox LAN) ----------------------------- # EVERY knob in cluster-vars' "LAN addressing" block is restated here EXPLICITLY, even where it # equals the base default. These values are what the LIVE dev cluster is running right now, so @@ -241,7 +183,7 @@ patches: # # Split-horizon DNS target: the Cilium-L2 VIP of the `shared` Gateway on the Proxmox LAN # (= CILIUM_LB_POOL_START, the pool's first address). coredns-custom answers - # *.dev.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. + # *.home.keithjackson.dev with this so in-cluster OIDC clients reach Keycloak via the Gateway. - op: replace path: /data/GATEWAY_VIP value: "192.168.1.53" diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index aaee0dd..630add8 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -129,23 +129,14 @@ patches: # # WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the # unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`). - # REASONING UPDATED 2026-09-04 for the new suffix direction (prod = the apex, dev = dev.home.), - # which changes WHY the blocker holds but not THAT it holds: - # 1. SELF-HOSTING CIRCULARITY — still true, and now the only reason. The Gitea this repo is - # pushed to runs ON THE DEV CLUSTER. Prod syncing from it would put prod's GitOps source - # inside the very cluster slated for decommission. Gitea must move to prod (or off-cluster) - # BEFORE dev is retired, or prod loses its source of truth. - # NOTE the suffix change makes the two Giteas DIFFERENT HOSTS: dev's is now - # gitea.dev.home.keithjackson.dev and prod's is gitea.home.keithjackson.dev. Prod pointing at - # the apex name therefore means "prod's OWN Gitea", never dev's — which is what makes the - # cutover below a one-line change once prod actually runs Gitea. - # 2. SPLIT-HORIZON DNS — NO LONGER A BLOCKER, and the note is kept because the old reasoning is - # easy to re-derive incorrectly. platform/coredns/coredns-custom.yaml answers the whole - # ${BASE_DOMAIN} zone with ${GATEWAY_VIP}; when prod's BASE_DOMAIN was `prod.home.` that meant - # prod resolved gitea.home.keithjackson.dev to PROD's gateway (192.168.2.20) where no Gitea - # listened — a misdirection. Now that prod IS the apex, resolving gitea.home.keithjackson.dev - # to prod's own Gateway is CORRECT: that is where prod's Gitea will be served from. What - # remains is purely reason 1 — the Gitea is not there YET. + # Two independent reasons, both consequences of the split-horizon design: + # 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's + # GitOps source lives inside the very cluster slated for decommission. Gitea must move to + # prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth. + # 2. SPLIT-HORIZON DNS SENDS IT TO THE WRONG PLACE. platform/coredns/coredns-custom.yaml + # answers the ENTIRE home.keithjackson.dev zone with ${GATEWAY_VIP}. On prod that is + # 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve + # gitea.home.keithjackson.dev to itself and never reach a repository. # # WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction): # bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the @@ -211,41 +202,35 @@ patches: path: /data/ACME_CLUSTER_ISSUER value: "letsencrypt-prod" # ========================================================================================== - # PROD DNS IDENTITY (R-DNS-01/02/03) — PROD **IS** THE APEX. + # PROD DNS IDENTITY (R-DNS-01/02/03) — the fix for the 2026-09-04 incident. # ========================================================================================== - # OPERATOR DECISION 2026-09-04, superseding the interim split committed earlier the same day - # (prod on `prod.home.` behind apex CNAMEs). The simpler target, and the one authored here: - # - # dev -> dev.home.keithjackson.dev (dev carries the label; clusters/dev) - # prod -> home.keithjackson.dev (THE APEX — this knob) - # - # There is NO `prod.` label and NO CNAME layer. `.home.keithjackson.dev` is prod's REAL - # name, so nothing has to alias anything: prod's Gitea is gitea.home.keithjackson.dev, its - # Keycloak keycloak.home.keithjackson.dev, and the wildcard cert is `*.home.keithjackson.dev`. - # - # BACKGROUND — WHY THE SUFFIXES HAD TO SPLIT AT ALL. Until 2026-09-04 every hostname was - # HARDCODED to home.keithjackson.dev in the base manifests, so prod was authored to serve the - # EXACT SAME public names as the live dev cluster out of the SAME Cloudflare zone. Two - # consequences, one of which already happened: + # PROD SERVES ITS OWN SUBDOMAIN. Until 2026-09-04 every hostname was HARDCODED to + # home.keithjackson.dev in the base manifests, so prod was authored to serve the EXACT SAME + # public names as the live dev cluster out of the SAME Cloudflare zone. Two consequences, one + # of which already happened: # 1. HAPPENED: prod's external-dns (policy: sync, owner id `default` — the same implicit # default dev had) concluded it owned dev's records and DELETED EVERY ONE of them. # 2. QUEUED BEHIND IT: platform-config would have requested a `*.home.keithjackson.dev` # wildcard from letsencrypt-prod for a cert dev already holds, and both clusters would # have raced each other's `_acme-challenge` TXT records on that one zone. + # Moving prod to its own suffix dissolves both: the two clusters can no longer name the same + # record. This is what makes ACME_CLUSTER_ISSUER: letsencrypt-prod above safe here. # - # *** THE ORDERING CONSTRAINT THIS VALUE CREATES — READ BEFORE UNSUSPENDING ANYTHING. *** - # Prod now claims the very names the LIVE dev cluster is serving today. Both hazards above are - # therefore still armed until DEV HAS VACATED THE APEX. Prod's `platform` and its 10 downstream - # tiers are SUSPENDED for exactly this reason. The gate is not "review is done", it is: - # (a) dev is reconciling on *.dev.home.keithjackson.dev with a Ready wildcard cert, AND - # (b) dev's old apex A/CNAME records and their TXT registry entries are GONE from Cloudflare - # (verified against Cloudflare directly — the workstation's dnsmasq is NOT evidence about public - # DNS; that mistake was made on 2026-09-04), AND - # (c) dev's old `*.home.keithjackson.dev` cert/Certificate is released, because prod ordering - # that same name set while dev's still exists counts against LE's duplicate-certificate - # limit (5 per week per identical name set). - # Full ordered runbook: infra/PROD-PROVISIONING.md §10. Unsuspending before (a)-(c) re-creates - # the 2026-09-04 incident with the roles reversed. + # dev -> home.keithjackson.dev (UNCHANGED — dev is live; phase 2 moves it) + # prod -> prod.home.keithjackson.dev (this knob) + # + # CONSEQUENCE — prod's hostnames are NOT the apex names. Prod's Gitea is + # gitea.prod.home.keithjackson.dev, its Keycloak keycloak.prod.home.keithjackson.dev, and so + # on; the wildcard cert becomes `*.prod.home.keithjackson.dev`. The apex CNAMEs that make + # `.home.keithjackson.dev` point at prod are PHASE 2, deliberately not done here — they + # would take traffic off the live dev cluster. infra/PROD-PROVISIONING.md §10. + # + # THE APEX INVERSION IS DELIBERATELY NOT ON THIS BRANCH. Moving prod to the bare apex (and dev + # to dev.home.) is a LIVE MIGRATION of the dev cluster, not a variable change — see + # infra/PROD-PROVISIONING.md §10a. It is kept as a single commit on branch + # `migration/dns-apex-inversion` (9137bbd) and is applied as step D3 of that runbook, with the + # operator present. Everything else on this branch is independent of it, which is the whole + # reason it was split out. # # [parity] MUST equal k3s_base_domain in infra/ansible/inventory/group_vars/k3s_prod/vars.yml. # The kube-apiserver's oidc-issuer-url is an INSTALL-TIME k3s flag, not a manifest, so it is @@ -253,33 +238,30 @@ patches: # trusts a different Keycloak than the one prod actually runs. - op: replace path: /data/BASE_DOMAIN - value: "home.keithjackson.dev" + value: "prod.home.keithjackson.dev" + # SAFETY NET #2 (independent of the owner id below): narrow ExternalDNS's name scope to prod's + # OWN subtree, so prod cannot create, update or delete any record outside it no matter what the + # TXT registry says. VERIFY ON FIRST RECONCILE — the Cloudflare provider uses this value for + # ZONE selection too, and on the version in use in 2026-08 a strict-subdomain filter dropped + # the parent zone ("no hosted zone matching record DNS Name"). If prod's external-dns logs no + # matching zone, it will create NOTHING — fail-closed, and it can still never delete dev's + # records. Fallback in that case: set this to "keithjackson.dev" (dev's value) and rely on + # EXTERNALDNS_TXT_OWNER_ID below, which is sufficient on its own. - op: replace path: /data/EXTERNALDNS_DOMAIN_FILTER - # MUST be the Cloudflare ZONE. Prod now serves the apex, so `home.keithjackson.dev` looks - # like a tighter, more honest filter than the zone — IT IS NOT, IT IS AN OUTAGE. FINDING - # F-DNS1 (DECISIONS-LOG 2026-08-17) proved a STRICT SUBDOMAIN of the hosted zone drops the - # zone entirely: ExternalDNS generates the record then skips it with "no hosted zone matching - # record DNS Name", so prod publishes NOTHING. An agent narrowed this to - # prod.home.keithjackson.dev on 2026-09-04 and it was reverted in review; do not retry it. - # Isolation from dev does NOT come from this filter — it comes from - # (1) the two clusters requesting DISJOINT hostname sets (apex vs dev.home.), and - # (2) txtOwnerId: prod vs dev, so neither registry ever claims the other's records. + # MUST be the Cloudflare ZONE, not prod's subdomain. FINDING F-DNS1 (DECISIONS-LOG, + # 2026-08-17) proved a subdomain filter DROPS THE PARENT ZONE: ExternalDNS generated the + # record then skipped it with "no hosted zone matching record DNS Name", so prod would + # publish NOTHING. Isolation from dev does NOT come from this filter — it comes from + # (1) prod only requesting *.prod.home.keithjackson.dev hostnames, and + # (2) txtOwnerId: prod, so prod's registry never claims dev's `default`-owned records. # That pairing is what the 2026-09-04 incident needed and this filter cannot provide. value: "keithjackson.dev" # SAFETY NET #1 — THE ACTUAL ROOT CAUSE OF THE INCIDENT. Both clusters left this unset and so # both inherited the chart default `default`; prod then treated dev's TXT-registry entries as - # its own and, under policy: sync, deleted the records behind them. - # - # PROD MUST NOT BE GIVEN `default`, however tempting it now looks. Prod is taking over the apex - # NAMES, and dev's existing apex records are registered under owner `default` — so `default` is - # the one value that would let prod adopt them. It would also let prod SYNC-DELETE the ones no - # prod source claims, on a cluster dev is still serving. Prod inherits the apex names; it must - # never inherit the apex OWNER ID. `prod` keeps prod fail-safe: records owned by `default` are - # invisible to it, so an early unsuspend cannot destroy dev's DNS the way 2026-09-04 did — it - # simply fails to publish until those records are retired, which is the correct behaviour. - # dev is `dev` (clusters/dev, changed with dev's migration); the base default is a - # non-colliding sentinel. All three differ. + # its own and, under policy: sync, deleted the records behind them. This value MUST differ from + # dev's (`default`, pinned in clusters/dev — dev's live records are registered under it and + # cannot be renamed without orphaning them). `prod` is distinct, which is the entire fix. - op: replace path: /data/EXTERNALDNS_TXT_OWNER_ID value: "prod" diff --git a/infra/PROD-PROVISIONING.md b/infra/PROD-PROVISIONING.md index f169050..f727d64 100644 --- a/infra/PROD-PROVISIONING.md +++ b/infra/PROD-PROVISIONING.md @@ -1177,7 +1177,7 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be | `192.168.1.55` (crafty) | `platform/crafty/service-l4.yaml` | prod-LAN address | | `192.168.1.54` (UniFi VM) | `apps/vms/unifi/50-service-l4.yaml` | prod-LAN address | | `192.168.1.9:9000` (S3 backup) | `platform/keycloak/postgres-cluster.yaml`, `apps/wave-0/{nextcloud,semaphore}-db.yaml` | reachable prod backup target | -| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real. Prod's value was inverted the same day to the APEX `home.keithjackson.dev`, with dev moving to `dev.home.keithjackson.dev` — see §10a, which is authored but NOT yet applied. | +| ~~`home.keithjackson.dev` (every hostname)~~ | ~~HTTPRoutes, Gateway, wildcard cert, coredns-custom, Keycloak realm, oauth2-proxy, Homepage, `catalog/`~~ | **CLOSED 2026-09-04 — see §10.** This one was not merely "would build wrong": it **already destroyed dev's public DNS**. `BASE_DOMAIN` was a declared-but-DEAD knob; it is now real, and prod serves `prod.home.keithjackson.dev`. | - [ ] **Tokenize each into a `cluster-vars` knob** with a base default, then set the value in *both* overlays. Adding a knob before referencing `${KNOB}` is the sanctioned mechanism (see the @@ -1331,12 +1331,10 @@ single-server-etcd window, and avoids a later member-add dance. specifically. - [ ] Monitoring and alerts live on prod. - [ ] The conformance suite passes green **against the prod cluster**, not just dev (R-TEST-03). - Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10a - "The L2 tests hardcode the apex". -- [ ] **Prod serves the apex DNS/TLS identity** and dev's records are provably untouched (§10/§10a): - distinct `EXTERNALDNS_TXT_OWNER_ID` (`prod` vs `dev`), prod wildcard cert - `*.home.keithjackson.dev`, dev migrated to `*.dev.home.keithjackson.dev` with its old apex - records retired (§10a step D8), and prod's own Keycloak realm + per-service client secrets. + Blocked on parameterizing the L2 tests that hardcode `home.keithjackson.dev` — see §10 phase 2. +- [ ] **Prod serves its own DNS/TLS identity** and dev's records are provably untouched (§10): + distinct `EXTERNALDNS_TXT_OWNER_ID`, prod wildcard cert `*.prod.home.keithjackson.dev`, and + prod's own Keycloak realm + per-service client secrets. ## 9a. PROD BRING-UP STATE (2026-09-04) — what came up, and the TWO blockers that need the operator @@ -1435,6 +1433,18 @@ unsuspending `platform` and re-queried after — **57 records, byte-for-byte ide entries still `owner=default`.** Snapshots are taken against the **Cloudflare API**, never a local resolver (a pfSense override produced a false healthy reading during the incident). +> ### THE APEX INVERSION IS NOT ON THIS BRANCH +> +> Sections 10 and 10a below describe the **target** DNS layout — prod on the bare apex, dev on +> `dev.home.` — and the ordered runbook for getting there. **That change is deliberately not authored +> on this branch.** It is one commit, `9137bbd`, kept on branch **`migration/dns-apex-inversion`**, +> because merging it *is* dev's migration: `clusters/dev/sync.yaml` reconciles `./clusters/dev` from +> `main` every 10 minutes, so a merge executes the cutover on the live dev cluster unattended. +> +> On this branch **prod is authored on `prod.home.keithjackson.dev`**, which is what prod actually +> runs. Read what follows as the plan, not as a description of the current authored state. The +> inversion is applied as step **D3** of the §10a runbook, operator present. + ## 10. DNS/TLS environment split — **the 2026-09-04 incident, and the target architecture** > **Current state (2026-09-04):** the target below is **AUTHORED, NOT APPLIED.** Prod's @@ -1473,82 +1483,43 @@ values; for DNS and TLS that promise was false, so `clusters/prod` was *authored public identity. Two clusters that describe the same DNS name will fight over it no matter how the DNS controller is configured. -### The architecture decision (operator) — **INVERTED 2026-09-04, this is the current target** +### The architecture decision (operator) ``` -dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev -prod -> *.home.keithjackson.dev gitea.home.keithjackson.dev (THE APEX) +dev -> *.dev.home.keithjackson.dev gitea.dev.home.keithjackson.dev +prod -> *.prod.home.keithjackson.dev gitea.prod.home.keithjackson.dev +apex -> CNAME .home.keithjackson.dev -> .prod.home.keithjackson.dev ``` -**Prod *is* the apex. There is no `prod.` label and no CNAME layer at all** — -`.home.keithjackson.dev` is prod's real name, not an alias for one. This **supersedes** the -interim split committed earlier the same day (`c8c1d29`), which put prod on -`*.prod.home.keithjackson.dev` behind apex CNAMEs pointing at it. That layer is deleted: fewer -moving parts, no CNAME chain to keep in step, and prod ends up on the names users already type. +### PHASE 1 — **prod only. Dev did not move.** (this change) -The cost of the inversion is that **dev must move**, and dev is the live cluster. The interim split -was chosen precisely because it moved only the unbuilt environment; this one does not have that -property, so the whole of §10's remaining risk now sits on dev's side, and it is sequenced below. +`BASE_DOMAIN` is now real: every hostname in every Flux-reconciled manifest derives from it. -### The values, per environment +| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | +|---|---|---|---| +| base default | `home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | +| **dev** | `home.keithjackson.dev` (**unchanged**) | `default` (**must not change**) | `keithjackson.dev` | +| **prod** | `prod.home.keithjackson.dev` | `prod` | `prod.home.keithjackson.dev` | -| | `BASE_DOMAIN` | `EXTERNALDNS_TXT_OWNER_ID` | `EXTERNALDNS_DOMAIN_FILTER` | Ansible `k3s_base_domain` | -|---|---|---|---|---| -| base default | `dev.home.keithjackson.dev` | `unset-see-cluster-vars` (non-colliding sentinel) | `keithjackson.dev` | `dev.home.keithjackson.dev` (role default) | -| **dev** | `dev.home.keithjackson.dev` | `dev` | `keithjackson.dev` | (inherits the role default) | -| **prod** | `home.keithjackson.dev` | `prod` | `keithjackson.dev` | `home.keithjackson.dev` (`group_vars/k3s_prod`) | +**Dev's effective hostname is deliberately still the apex `home.keithjackson.dev`.** Moving prod +alone ends the collision and unblocks prod, and it is the whole of the risk that had to be taken +while the operator is away. Dev's rendered output is **byte-identical** except for one added line, +`txtOwnerId: default`, which is the value dev's external-dns was already using implicitly — verified +by rendering all 12 `substituteFrom` paths through `kubectl kustomize | flux envsubst --strict` for +both overlays and diffing. -`[parity]`: `k3s_base_domain` must equal the same cluster's `BASE_DOMAIN`. The kube-apiserver's -`oidc-issuer-url` is a k3s **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux -post-build substitution can never reach it — hence one fact in two places. If they disagree the -apiserver validates tokens against a different Keycloak than the cluster runs: kubectl SSO fails -while the cert-based admin kubeconfig keeps working, so it presents as "SSO is broken", not as skew. +**Two independent guards now, either sufficient alone:** -**`EXTERNALDNS_DOMAIN_FILTER` is the Cloudflare ZONE for BOTH, and must stay that way.** FINDING -F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for **zone selection** as -well as record matching, so a filter that is a strict subdomain of the hosted zone matches **no -zone** — external-dns generates the record and then skips it with *"no hosted zone matching record -DNS Name"*, publishing **nothing**. This has been "hardened" and reverted more than once. It is not -defence in depth; it is a total outage of DNS publication. Isolation comes from the two guards below, -which a domain filter cannot provide anyway: both environments live in the one zone regardless. +1. **`txtOwnerId`** — per-environment identity. A cluster only ever touches records it owns. This is + the direct fix for the incident. Dev **must** render exactly `default`: its live records are + already registered under that owner, and any other value would make dev's own external-dns stop + recognising them and re-create duplicates beside the orphans. +2. **`domainFilters`** — per-environment name scope. Prod cannot see a name outside its own subtree. + Defence in depth, added because the first guard is a single string in a single file. -**The two guards that actually work:** - -1. **`txtOwnerId`, per-environment identity** — a cluster only ever reconciles records its own owner - id claims in the TXT registry. This is the direct fix for the incident. -2. **Disjoint hostname sets** — after the migration the two clusters never describe the same record, - so there is nothing to fight over. *This guard does not exist during the migration window*, which - is the entire reason the ordering below is mandatory. - -#### Why dev's owner id changed from `default` to `dev` - -Until this migration dev **had** to render `default`: its live apex records were already registered -in Cloudflare's TXT registry under that owner, and renaming would have orphaned every one of them. -**The migration dissolves that constraint** — on `*.dev.home.keithjackson.dev` dev publishes an -entirely new record set with no prior registry entries to recognise, so the id is free and is chosen -on its merits. `default` has none: it is the value external-dns uses when you **forget** the flag, -which is exactly how both clusters came to share it, and an environment whose identity is the -forget-value can never be told apart from a misconfigured newcomer. Consequences, both deliberate: - -- Dev's **old apex records stay owned by `default`**, which no environment now claims, so nothing - deletes them implicitly on some future reconcile. Retiring them is an explicit step (**D7**) and - prod's first publish is gated on that retirement being *confirmed* — not on two reconcile loops - racing each other. -- **Prod stays `prod` and must never be given `default`.** Prod is taking over the apex *names*, and - dev's existing apex records are registered under `default` — so `default` is the one value that - would let prod adopt them, and under `policy: sync` delete the ones no prod source claims, on a - cluster dev is still serving. That is the 2026-09-04 incident re-armed with the roles swapped. - Prod inherits the apex **names**; it must never inherit the apex **owner id**. - -All three values differ, and the base default remains a sentinel so an environment that forgets the -knob owns nothing and can therefore delete nothing. - -### PHASE 1 — prod tokenized and moved off dev's names (`c8c1d29`, done; suffix since inverted) - -`BASE_DOMAIN` was a **dead knob** — declared in `clusters/base/cluster-vars.yaml` and referenced by -no reconciled manifest, because every hostname was a hardcoded literal. `c8c1d29` made it real (141 -references across the reconciled set) and gave prod its own suffix. That work stands; only the -*value* prod holds has changed, from `prod.home.keithjackson.dev` to the apex. +The base default for the owner id is a **non-colliding sentinel, deliberately not `default`**: a +future environment that forgets the knob must not silently inherit dev's identity — that is exactly +what happened here. With the sentinel such a cluster owns nothing, so it can delete nothing. ### What could NOT be tokenized, and why @@ -1557,49 +1528,49 @@ references across the reconciled set) and gave prod its own suffix. That work st Re-encrypting needs the operator's age key, rewrites the MAC over every value in the file, and **cannot be verified by the offline render harness** (`kustomize` does not decrypt), so a mistake would break Headlamp SSO on the live dev cluster with no pre-flight proof. Left as-is deliberately. - **Both environments now need action:** dev's copy must move to `keycloak.dev.home.keithjackson.dev` - as part of step **D5**, and prod needs its own copy with `keycloak.home.keithjackson.dev` and a - client secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client - secret is realm-scoped. + **Prod action required:** prod's Headlamp will otherwise authenticate against **dev's** Keycloak. + Prod needs its own copy of this Secret with `keycloak.prod.home.keithjackson.dev` and a client + secret from prod's own realm — which it needs regardless, since every `*.sops.yaml` client secret + is per-realm. - **`oidc-issuer-url` in the kube-apiserver flags** (`roles/k3s_server/tasks/main.yml`) — a k3s - server flag, not a manifest. Handled on the **Ansible** seam instead: role default - `k3s_base_domain` (dev's value) + derived `k3s_oidc_issuer_url`, overridden for prod in - `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`, - so a fourth prod server inherits it from group membership. Verified per host by a read-only probe - (`connection: local`, zero SSH, `changed=0`): all three dev servers resolve - `https://keycloak.dev.home.keithjackson.dev/realms/homelab`, all three prod servers resolve - `https://keycloak.home.keithjackson.dev/realms/homelab`. -- **`spec.url` of dev's `GitRepository`** (`clusters/dev/kustomization.yaml`) — see the strand below. - It *cannot* be tokenized: the `flux-system` root Kustomization that renders `./clusters/dev` - declares no `postBuild.substituteFrom`, so `gitea.${BASE_DOMAIN}` there would ship the **literal - token** and break the source outright (the L0 `flux-postbuild-substitution` check asserts exactly - this). It is a hand-maintained literal that must move in lockstep with `BASE_DOMAIN`. + **server flag** in `/etc/rancher/k3s/config.yaml`, not a manifest, so Flux post-build substitution + can never reach it. Correctly tokenized on the **Ansible** side instead: new role default + `k3s_base_domain` (dev value) + derived `k3s_oidc_issuer_url`, overridden for prod in + `inventory/group_vars/k3s_prod/vars.yml` — the same inventory seam as `k3s_data_dir`/`k3s_api_host`. + `[parity]`: `k3s_base_domain` **must** equal the same cluster's `BASE_DOMAIN`, or the apiserver + validates tokens against a different Keycloak than the cluster runs and kubectl SSO fails while the + cert-based admin kubeconfig keeps working (so it looks like "SSO is broken", not like config skew). ---- +### A pre-existing prod problem this incidentally FIXED -## 10a. THE MIGRATION RUNBOOK — ordering is load-bearing +§4 recorded that prod nodes use **public** resolvers, so the kube-apiserver would resolve its OIDC +issuer publicly and get **dev's** Gateway VIP (192.168.1.53) — the wrong cluster on the wrong LAN, +making prod kubectl SSO impossible, and requiring an `/etc/hosts` pin as a workaround. With prod on +its own subdomain the public record for `keycloak.prod.home.keithjackson.dev` is published by +**prod's own** external-dns pointing at **prod's** Gateway VIP (192.168.2.20), so the public answer +and the split-horizon answer agree. **No `/etc/hosts` pin is needed.** This holds only once prod's +external-dns has actually published the record; until then use the break-glass cert kubeconfig. -> **Prod's `platform` tier and its 10 downstream tiers are SUSPENDED and must stay suspended until -> step P1.** Nothing below is safe to run out of order, and nothing below should be run while the -> operator is away. +### Verify on prod's first reconcile -### The two ways this goes wrong +- [ ] **`domainFilters` zone selection.** The Cloudflare provider uses this value for *zone* lookup + as well as record matching, and in 2026-08 a strict-subdomain filter dropped the parent zone + ("no hosted zone matching record DNS Name"). If prod's external-dns logs no matching zone it + will create **nothing** — fail-closed, never destructive. Fallback: set prod's + `EXTERNALDNS_DOMAIN_FILTER` back to `keithjackson.dev` and rely on `txtOwnerId`, which is + sufficient on its own. +- [ ] **Prod's wildcard cert is `*.prod.home.keithjackson.dev`** and solves DNS-01 without touching + dev's `_acme-challenge` records. +- [ ] **Dev's records are untouched** after prod's tiers are resumed. Confirm dev's TXT registry + entries still read owner `default` and that prod's read owner `prod`. +- [ ] **Prod's own Keycloak realm + per-service client secrets** exist (every `*.sops.yaml` client + secret is realm-scoped; prod cannot reuse dev's). -**(1) BOTH CLUSTERS ON THE APEX — the 2026-09-04 incident, recreated.** If prod publishes -`*.home.keithjackson.dev` before dev has vacated it, two clusters describe the same records in the -same zone again. Distinct `txtOwnerId`s stop prod from *deleting* dev's records — that guard holds — -but they do not stop the second failure mode: both clusters requesting a `letsencrypt-prod` wildcard -for the **same name set** and racing each other's `_acme-challenge` TXT records, so neither reliably -solves DNS-01. **Rule: dev vacates the apex first, and the vacate is verified against Cloudflare -before prod is unsuspended.** +### PHASE 2 — dev migration + apex CNAMEs (**deferred, not done**) -**(2) THE STRAND — dev's GitOps source is the thing that moves.** `spec.url` of dev's -`GitRepository` is `https://gitea.home.keithjackson.dev/...`, which is **both** dev's GitOps source -**and** the PM's push target. Flipping dev's `BASE_DOMAIN` moves Gitea's HTTPRoute, its public DNS -record and the wildcard cert covering it to `gitea.dev.home.keithjackson.dev` all at once. If that -host does not already resolve and serve a valid, Ready cert at that instant, **dev stops reconciling -and cannot be fixed by pushing** — the broken thing *is* the path Flux uses to read pushes. Recovery -is then hands-on and needs LAN access to the cluster. +Deliberately not attempted here: dev is **live** (18 Longhorn volumes, working SSO) and it hosts the +Gitea that is currently this repo's GitOps source. Moving it is a migration, not a variable change. +Required, roughly in order: **(2b) AND THE MERGE ITSELF IS THE TRIGGER.** `clusters/dev/sync.yaml` is a self-managing root Kustomization reconciling `./clusters/dev` from **`main`** every 10 minutes, and prod tracks `main` diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index 820f110..bca8337 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -306,47 +306,35 @@ node_baseline_dns_servers: # k3s_base_domain, below). It reaches the issuer through the NODE resolver, i.e. through # these PUBLIC upstreams — not through CoreDNS's split-horizon answer. # - # ORDERING HAZARD, NOT YET CLEARED. Prod's issuer is keycloak.home.keithjackson.dev — an APEX - # name, because prod IS the apex under the 2026-09-04 inversion (k3s_base_domain below). - # That is the name the LIVE DEV CLUSTER answers today, and its public A record currently - # points at DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. A prod - # node resolving its own issuer through these public upstreams therefore reaches dev until: - # (1) dev has migrated to *.dev.home.keithjackson.dev and RETIRED its apex records, and - # (2) PROD's own external-dns has published keycloak.home.keithjackson.dev at PROD's - # Gateway VIP (192.168.2.20). - # Only then do the public answer and the in-cluster split-horizon answer AGREE, and no - # /etc/hosts pin is needed. Until then kubectl OIDC SSO on prod cannot work; the cert-based - # admin kubeconfig is the break-glass path and always works. Ordering, not a design gap — - # the ordered steps are infra/PROD-PROVISIONING.md §10. + # THIS USED TO BE BROKEN, AND THE 2026-09-04 SUBDOMAIN SPLIT FIXED IT. While prod was + # authored to serve dev's hostnames, a prod node resolving keycloak.home.keithjackson.dev + # publicly got DEV's Gateway VIP (192.168.1.53) — the wrong cluster, on the wrong LAN, so + # kubectl OIDC SSO on prod could not work at all. Now prod's issuer is + # keycloak.prod.home.keithjackson.dev, and the public record for that name is published by + # PROD's own external-dns pointing at PROD's Gateway VIP (192.168.2.20). The public answer + # and the split-horizon answer therefore AGREE, and no /etc/hosts pin is needed. + # + # STILL TO VERIFY on prod bring-up (PROD-PROVISIONING.md §10): this holds only once prod's + # external-dns has actually published the record. Until then the issuer does not resolve + # from the nodes and kubectl SSO fails — the cert-based admin kubeconfig is the break-glass + # path, and it always works. Ordering, not a design gap. node_baseline_dns_search: [] # --- platform hostname suffix, ANSIBLE SIDE (R-ENV-02, R-DNS-01) -------------------- -# PROD **IS** THE APEX. [parity] MUST equal BASE_DOMAIN in clusters/prod/kustomization.yaml — -# the same one-fact-in-two-places contract as k3s_data_dir and k3s_api_host. The Flux side -# cannot supply this: the kube-apiserver's oidc-issuer-url is a k3s server FLAG written to -# /etc/rancher/k3s/config.yaml, not a manifest, so postBuild.substituteFrom never sees it. -# Full rationale in roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url derives. +# PROD SERVES ITS OWN SUBDOMAIN. [parity] MUST equal BASE_DOMAIN in clusters/prod/ +# kustomization.yaml — the same one-fact-in-two-places contract as k3s_data_dir and +# k3s_api_host. The Flux side cannot supply this: the kube-apiserver's oidc-issuer-url is a +# k3s server FLAG written to /etc/rancher/k3s/config.yaml, not a manifest, so +# postBuild.substituteFrom never sees it. Full rationale in +# roles/k3s_server/defaults/main.yml, where k3s_oidc_issuer_url is derived from this. # -# WHY THE SUFFIXES SPLIT (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve -# the SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns -# deleted every one of dev's public records. -# -# WHICH WAY ROUND — INVERTED 2026-09-04 by operator decision, superseding the interim split -# committed earlier the same day (prod on `prod.home.` behind apex CNAMEs). The simpler target: -# prod -> home.keithjackson.dev (THE APEX; this variable — no `prod.` label, no CNAMEs) -# dev -> dev.home.keithjackson.dev (roles/k3s_server/defaults/main.yml) -# So prod's Keycloak issuer is https://keycloak.home.keithjackson.dev/realms/homelab, and the -# `kubernetes` OIDC client in PROD's OWN Keycloak realm is what prod's kubectl SSO authenticates -# against. -# -# ORDERING HAZARD SPECIFIC TO THIS VARIABLE. keycloak.home.keithjackson.dev is the name the LIVE -# dev cluster answers today, and prod's nodes resolve the issuer through PUBLIC upstreams (see -# node_baseline_dns_search above). So until dev has vacated the apex and PROD's external-dns has -# published that name at PROD's Gateway VIP (192.168.2.20), a prod node resolving its own issuer -# gets DEV's VIP (192.168.1.53) — the wrong cluster, on the wrong LAN. kubectl OIDC SSO on prod -# therefore does not work until §10's dev-vacate steps are complete; the cert-based admin -# kubeconfig is the break-glass path and always works. Ordering, not a design gap. -k3s_base_domain: "home.keithjackson.dev" +# WHY PROD MOVED (infra/PROD-PROVISIONING.md §10): both clusters were authored to serve the +# SAME hostnames from the SAME Cloudflare zone, and on 2026-09-04 prod's external-dns +# deleted every one of dev's public records. Dev stays on home.keithjackson.dev (it is LIVE); +# prod takes prod.home.keithjackson.dev. Consequence here: prod's Keycloak issuer is +# https://keycloak.prod.home.keithjackson.dev/realms/homelab, so the `kubernetes` OIDC client +# in prod's OWN Keycloak realm is what prod's kubectl SSO authenticates against. +k3s_base_domain: "prod.home.keithjackson.dev" # --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) ----- # roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml index eef431e..31e950f 100644 --- a/infra/ansible/roles/k3s_server/defaults/main.yml +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -95,26 +95,15 @@ k3s_tls_sans: >- # while the cert-based admin kubeconfig keeps working — so it looks like "SSO is broken", # not like a config skew. The break-glass cert kubeconfig is why this is recoverable. # -# THE DEFAULT IS THE DEV VALUE — the same convention as k3s_data_dir above and cluster-vars' -# GATEWAY_VIP / LONGHORN_DATA_PATH: the role default is dev's value and prod overrides it in -# inventory/group_vars/k3s_prod/vars.yml (the same inventory seam k3s_data_dir / k3s_api_host / -# kube_vip_address use, so a FOURTH prod server inherits it from group membership rather than -# from anyone remembering). +# THE DEFAULT IS THE DEV VALUE, restated explicitly rather than left implicit — the same +# convention as k3s_data_dir above and cluster-vars' GATEWAY_VIP / LONGHORN_DATA_PATH: the +# base default is the known-good LIVE value, so dev's rendered config.yaml is byte-for-byte +# what it already is and the Restart handler does NOT fire on a re-run. # -# *** WHICH DEV VALUE — UPDATED 2026-09-04, AND IT IS AHEAD OF THE LIVE DEV CLUSTER. *** -# The operator inverted the suffix split: PROD is now the apex `home.keithjackson.dev` and DEV -# carries the label `dev.home.keithjackson.dev`. This default therefore moved from the apex to -# dev's subdomain, mirroring BASE_DOMAIN in clusters/dev/kustomization.yaml. -# -# CONSEQUENCE — this default no longer matches what the live dev nodes are running, so unlike -# every previous edit to this file A DEV RE-RUN IS NOT A NO-OP. Rendering config.yaml with the -# new issuer CHANGES the file, FIRES the Restart handler, and rolls k3s on all three dev -# servers, which also invalidates every user's OIDC kubeconfig (SSO must be re-issued). That is -# a deliberate step of the dev DNS migration, NOT a side effect to discover during unrelated -# work: DO NOT run the k3s_server role against dev until step D6 of -# infra/PROD-PROVISIONING.md §10, and never while the operator is away. The break-glass -# cert-based admin kubeconfig keeps working throughout, which is what makes the step safe. -k3s_base_domain: "dev.home.keithjackson.dev" +# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same inventory seam +# k3s_data_dir / k3s_api_host / kube_vip_address use, so a FOURTH prod server inherits it +# from group membership rather than from anyone remembering. +k3s_base_domain: "home.keithjackson.dev" # Derived, so the realm path is stated once. Override k3s_base_domain, not this. k3s_oidc_issuer_url: "https://keycloak.{{ k3s_base_domain }}/realms/homelab" diff --git a/platform/externaldns/helmrelease.yaml b/platform/externaldns/helmrelease.yaml index 4324681..3a56c3b 100644 --- a/platform/externaldns/helmrelease.yaml +++ b/platform/externaldns/helmrelease.yaml @@ -39,33 +39,24 @@ spec: # garbage-collected them. `policy: sync` is correct and stays; what was missing was identity. # Full write-up: infra/PROD-PROVISIONING.md §10. # - # THE TWO GUARDS THAT ACTUALLY PREVENT A RECURRENCE — note that neither is domainFilters: - # 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns. - # 2. DISJOINT HOSTNAME SETS — the two environments serve different suffixes, so they never - # describe the same record in the first place. dev = *.dev.home.keithjackson.dev, - # prod = *.home.keithjackson.dev (prod IS the apex; operator decision 2026-09-04). + # TWO INDEPENDENT GUARDS NOW, either of which alone prevents a recurrence: + # 1. txtOwnerId — per-environment identity; a cluster only ever touches records it owns. + # 2. domainFilters — per-environment name scope; prod cannot even see names outside its + # own subtree. Defence in depth, added after the incident. # --------------------------------------------------------------------------------------------- - # Which names this instance may manage at all. BOTH ENVIRONMENTS RENDER THE CLOUDFLARE **ZONE** - # `keithjackson.dev` HERE, and this is NOT an oversight to be tightened. - # - # FINDING F-DNS1 (DECISIONS-LOG 2026-08-17): the Cloudflare provider uses this value for ZONE - # SELECTION as well as record matching, so a filter that is a STRICT SUBDOMAIN of the hosted zone - # matches NO zone — external-dns generates the record and then skips it with "no hosted zone - # matching record DNS Name", publishing NOTHING. This has been "improved" and reverted more than - # once (an agent narrowed prod to prod.home.keithjackson.dev on 2026-09-04; overridden in - # review). It is not defence in depth; it is a total outage of DNS publication. Isolation comes - # from the two guards above, which a domain filter cannot provide anyway — both environments live - # in the ONE zone regardless of what this says. See clusters/base/cluster-vars.yaml. + # Which names this instance may manage at all. Cloudflare-provider caveat, empirically hit + # 2026-08: this value is used for ZONE selection as well as record matching, and a filter that is + # a strict subdomain of the zone dropped the parent zone entirely ("no hosted zone matching record + # DNS Name") — so DEV keeps the zone apex `keithjackson.dev`, which is also what dev serves the + # apex of. PROD narrows to `prod.home.keithjackson.dev`. If prod's first reconcile logs no + # matching zone, that caveat still applies: prod creates NOTHING (fail-closed, never destructive) + # and the fallback is to widen prod's EXTERNALDNS_DOMAIN_FILTER back to the zone apex, leaving + # txtOwnerId as the sole guard. See clusters/base/cluster-vars.yaml. domainFilters: - "${EXTERNALDNS_DOMAIN_FILTER}" # The TXT-registry owner id. MUST be distinct per environment — this is the knob whose absence - # caused the incident. dev renders `dev`, prod renders `prod`, and the BASE default is a - # non-colliding sentinel so a new environment that forgets the knob owns nothing and can delete - # nothing. NOTHING renders `default` any more (the external-dns chart default, and the value both - # clusters silently shared): dev held it only because its live apex records were registered under - # it, and dev's move to *.dev.home.keithjackson.dev retired that constraint along with those - # records. Prod must NOT be given `default` even though it inherits the apex NAMES — that would - # hand prod ownership of dev's old apex records and let `policy: sync` below delete them. + # caused the incident. Dev renders `default` (its live records are already registered under that + # owner; changing it would orphan them and duplicate every record). Prod renders `prod`. txtOwnerId: "${EXTERNALDNS_TXT_OWNER_ID}" policy: sync # create + update + delete records as services come and go — but ONLY # records this owner id owns AND this domainFilter admits (see above).