Merge fix/split-horizon-dns: in-cluster split-horizon DNS (R-DNS-01/03) — unblocks SSO

clusters/base/cluster-vars.yaml

@@ -21,6 +21,13 @@ data:
   DNS_ZONE: "keithjackson.dev"              # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it);
                                             # ExternalDNS domain-filter must be the zone, not the subdomain
   ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
+  GATEWAY_VIP: "10.73.0.240"                 # in-cluster split-horizon target (R-DNS-01/03): the L2-announced
+                                            # ingress VIP of the `shared` Gateway (first IP of the Cilium
+                                            # LB IP pool 10.73.0.240-.250, platform-config/cilium/l2-ip-pool.yaml).
+                                            # coredns-custom answers *.home.keithjackson.dev with THIS so pod->
+                                            # public-hostname traffic hairpins through the Gateway (correct SNI ->
+                                            # wildcard cert). Base default = dev VIP; a prod overlay must set the
+                                            # prod LAN VIP alongside the prod L2 pool range when hardware is built.
 
   # --- storage (ADR-0003) ---
   LONGHORN_DEFAULT_REPLICA_COUNT: "3"       # default StorageClass replica count (3 on 3 nodes)

clusters/prod/kustomization.yaml

@@ -45,3 +45,7 @@ patches:
         path: /data/VELERO_RETENTION
         value: "720h"
       # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist.
+      # TODO(WS-NET, Phase 7): set GATEWAY_VIP to the prod LAN ingress VIP (add a `replace` op here)
+      #   at the same time the prod Cilium LB IP pool range is supplied — the split-horizon
+      #   coredns-custom (platform/coredns) answers *.home.keithjackson.dev with ${GATEWAY_VIP}, so
+      #   the prod value must match the prod pool's Gateway address. Base default is the dev VIP.

conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml

@@ -0,0 +1,88 @@
+# =============================================================================
+# L2 BEHAVIORAL — split-horizon INTERNAL resolution  (R-DNS-01 / R-DNS-03)   [RUNNABLE]
+# Catalog: docs/14 "DNS/exposure (R-DNS-*)"
+# Spec:    docs/09 "Split-horizon resolution (R-DNS-03)" — internal clients resolve
+#          <svc>.home.keithjackson.dev to the INTERNAL Gateway VIP, not the public hairpin.
+# Impl:    platform/coredns/coredns-custom.yaml (k3s CoreDNS `import *.server`; template plugin
+#          answers the whole home.keithjackson.dev zone with ${GATEWAY_VIP}).
+#
+# GIVEN the split-horizon coredns-custom ConfigMap is reconciled into k3s CoreDNS,
+# WHEN  an in-cluster pod resolves a platform hostname over its PUBLIC name
+#       (keycloak.home.keithjackson.dev) using the default cluster resolver (CoreDNS 10.43.0.10),
+# THEN  the answer is the in-cluster Gateway VIP (the L2-announced LB range 10.73.0.24x) — NOT a
+#       public/off-cluster IP. This is what lets pod->public-hostname traffic hairpin through the
+#       Gateway with the correct SNI (*.home.keithjackson.dev) so the wildcard cert validates
+#       (R-DNS-01 "services reachable at <svc>.home..."; R-DNS-03 split-horizon).
+#
+# Complements dns-exposure (R-DNS-02, the EXTERNAL/public-record side). Together they cover the
+# split-horizon pair: external answer = public record (R-DNS-02); internal answer = Gateway VIP
+# (R-DNS-01/03).
+#
+# CLEANUP: chainsaw deletes the per-test namespace (Job included) at teardown. Read-only otherwise.
+# chainsaw v0.2.12: Job + status assert; no test-level finally.
+# =============================================================================
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: dns-internal-resolution
+  labels:
+    conformance.home.k3s/R-DNS-01: "true"
+    conformance.home.k3s/R-DNS-03: "true"
+  annotations:
+    conformance.home.k3s/requirements: "R-DNS-01,R-DNS-03"
+    conformance.home.k3s/layer: "L2"
+    conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
+    conformance.home.k3s/intent: >-
+      An in-cluster pod resolving keycloak.home.keithjackson.dev via cluster DNS gets the internal
+      Gateway VIP (10.73.0.24x), not the public hairpin — split-horizon in-cluster resolution.
+spec:
+  timeouts:
+    apply: 60s
+    assert: 2m
+    exec: 2m
+  steps:
+    - name: then-cluster-dns-answers-with-the-internal-gateway-vip
+      try:
+        - apply:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: dns-internal-probe
+              spec:
+                backoffLimit: 2
+                activeDeadlineSeconds: 100
+                template:
+                  spec:
+                    restartPolicy: Never
+                    containers:
+                      - name: probe
+                        image: busybox:1.36.1
+                        command:
+                          - sh
+                          - -c
+                          - |
+                            set -u
+                            host="keycloak.home.keithjackson.dev"
+                            # Resolve via the DEFAULT cluster resolver (CoreDNS, /etc/resolv.conf ->
+                            # 10.43.0.10). Split-horizon must answer with the in-cluster Gateway VIP
+                            # (L2 pool 10.73.0.24x), proving pods hairpin through the Gateway.
+                            out="$(nslookup "$host" 2>/dev/null)"
+                            echo "$out"
+                            ip="$(echo "$out" | awk '/^Address: /{print $2}' | grep -E '^10\.73\.0\.24[0-9]$' | head -1)"
+                            if [ -n "$ip" ]; then
+                              echo "INTERNAL_RESOLUTION_OK $host -> $ip (in-cluster Gateway VIP)"
+                              exit 0
+                            fi
+                            echo "FAIL: $host did not resolve to the in-cluster Gateway VIP (10.73.0.24x);"
+                            echo "      split-horizon coredns-custom not in effect (got a public/off-cluster answer)."
+                            exit 1
+        - assert:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: dns-internal-probe
+              status:
+                succeeded: 1

platform/coredns/coredns-custom.yaml

@@ -0,0 +1,45 @@
+---
+# Split-horizon DNS for cluster pods (R-DNS-01 / R-DNS-03).
+#
+# PROBLEM: an in-cluster client resolving a platform hostname over its PUBLIC name
+# (e.g. `keycloak.home.keithjackson.dev`) gets the off-cluster/public answer — it does NOT reach the
+# in-cluster Gateway. That path does not serve the `*.home.keithjackson.dev` SNI, so TLS fails with
+# `remote error: tls: unrecognized name` (oauth2-proxy OIDC discovery, Grafana generic_oauth, etc.).
+#
+# FIX (split-horizon, docs/09 "Split-horizon resolution"): inside the cluster, answer the whole
+# `home.keithjackson.dev` zone with the L2-announced ingress VIP of the `shared` Gateway. Pod ->
+# public-hostname traffic then hairpins through the Gateway with the correct SNI -> wildcard cert ->
+# works. External/LAN clients keep their own answers (PiHole split-horizon, docs/09) — this ConfigMap
+# only affects CLUSTER DNS (CoreDNS), not LAN DNS.
+#
+# MECHANISM (k3s-specific, F-DNS / docs/09): k3s CoreDNS ships a Corefile with
+# `import /etc/coredns/custom/*.server`, and mounts the `coredns-custom` ConfigMap (kube-system) at
+# /etc/coredns/custom. Each `*.server` key becomes a top-level server block imported into the
+# Corefile. CoreDNS `reload` picks up ConfigMap changes automatically — NO pod restart needed
+# (GitOps-friendly; the change reconciles live).
+#
+# ${GATEWAY_VIP} is substituted by Flux postBuild.substituteFrom cluster-vars (R-ENV-02): dev = the
+# dev L2 pool VIP, prod = the prod LAN VIP. The unified graph is identical across environments; only
+# the cluster-var differs. See clusters/base/cluster-vars.yaml.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  # Filename `home.server` -> imported as a standalone server block for the whole zone.
+  home.server: |
+    home.keithjackson.dev:53 {
+        errors
+        # Answer EVERY name in the zone (apex + all `<svc>.home.keithjackson.dev`) with the Gateway
+        # VIP. The template plugin's default match is `.*`, so this covers current and FUTURE app
+        # hostnames without editing this file per service. `{{ .Name }}` echoes the queried FQDN.
+        template IN A {
+            answer "{{ .Name }} 60 IN A ${GATEWAY_VIP}"
+        }
+        # AAAA -> NODATA (NOERROR, no answer): the name has an A but no AAAA, so IPv6-capable clients
+        # correctly fall back to the A record instead of getting NXDOMAIN/SERVFAIL.
+        template IN AAAA {
+            rcode NOERROR
+        }
+    }

platform/coredns/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+# platform/coredns — in-cluster split-horizon DNS (R-DNS-01 / R-DNS-03).
+# A single `coredns-custom` ConfigMap in kube-system that k3s CoreDNS imports as an extra server
+# block (see coredns-custom.yaml). Rendered by the `platform` Flux Kustomization, which supplies
+# ${GATEWAY_VIP} via substituteFrom cluster-vars (R-ENV-02).
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - coredns-custom.yaml

platform/kustomization.yaml

@@ -17,6 +17,7 @@ resources:
   - cert-manager       # Let's Encrypt DNS-01 wildcard — needs SOPS `cloudflare-api-token`
   - externaldns        # auto DNS records — needs SOPS `cloudflare-api-token`
   - gateway            # Gateway API listeners (ingress; replaces Traefik)
+  - coredns            # split-horizon in-cluster DNS: *.home.keithjackson.dev -> Gateway VIP (R-DNS-01/03)
   # ---- Phase 3 — Egress / VPN (ADR-0015): node-level host OpenVPN split-tunnel (Ansible role
   #      vpn_egress_gateway) steered by a Cilium Egress Gateway policy, with a host nftables kill-switch.
   #      No in-cluster gateway pod (Gluetun retired). See ADR-0015 + DECISIONS-LOG "Phase 3 VPN". ----