diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 5508c31..b2f0bf9 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -21,6 +21,13 @@ data: DNS_ZONE: "keithjackson.dev" # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it); # ExternalDNS domain-filter must be the zone, not the subdomain ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006) + GATEWAY_VIP: "10.73.0.240" # in-cluster split-horizon target (R-DNS-01/03): the L2-announced + # ingress VIP of the `shared` Gateway (first IP of the Cilium + # LB IP pool 10.73.0.240-.250, platform-config/cilium/l2-ip-pool.yaml). + # coredns-custom answers *.home.keithjackson.dev with THIS so pod-> + # public-hostname traffic hairpins through the Gateway (correct SNI -> + # wildcard cert). Base default = dev VIP; a prod overlay must set the + # prod LAN VIP alongside the prod L2 pool range when hardware is built. # --- storage (ADR-0003) --- LONGHORN_DEFAULT_REPLICA_COUNT: "3" # default StorageClass replica count (3 on 3 nodes) diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index b0de8e9..c0902bd 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -45,3 +45,7 @@ patches: path: /data/VELERO_RETENTION value: "720h" # TODO(WS-INFRA, Phase 7): pin prod node placement once the bare-metal role labels exist. + # TODO(WS-NET, Phase 7): set GATEWAY_VIP to the prod LAN ingress VIP (add a `replace` op here) + # at the same time the prod Cilium LB IP pool range is supplied — the split-horizon + # coredns-custom (platform/coredns) answers *.home.keithjackson.dev with ${GATEWAY_VIP}, so + # the prod value must match the prod pool's Gateway address. Base default is the dev VIP. diff --git a/conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml b/conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml new file mode 100644 index 0000000..2eee4d9 --- /dev/null +++ b/conformance/tests/l2-behavioral/dns-internal-resolution/chainsaw-test.yaml @@ -0,0 +1,88 @@ +# ============================================================================= +# L2 BEHAVIORAL — split-horizon INTERNAL resolution (R-DNS-01 / R-DNS-03) [RUNNABLE] +# Catalog: docs/14 "DNS/exposure (R-DNS-*)" +# Spec: docs/09 "Split-horizon resolution (R-DNS-03)" — internal clients resolve +# .home.keithjackson.dev to the INTERNAL Gateway VIP, not the public hairpin. +# Impl: platform/coredns/coredns-custom.yaml (k3s CoreDNS `import *.server`; template plugin +# answers the whole home.keithjackson.dev zone with ${GATEWAY_VIP}). +# +# GIVEN the split-horizon coredns-custom ConfigMap is reconciled into k3s CoreDNS, +# WHEN an in-cluster pod resolves a platform hostname over its PUBLIC name +# (keycloak.home.keithjackson.dev) using the default cluster resolver (CoreDNS 10.43.0.10), +# THEN the answer is the in-cluster Gateway VIP (the L2-announced LB range 10.73.0.24x) — NOT a +# public/off-cluster IP. This is what lets pod->public-hostname traffic hairpin through the +# Gateway with the correct SNI (*.home.keithjackson.dev) so the wildcard cert validates +# (R-DNS-01 "services reachable at .home..."; R-DNS-03 split-horizon). +# +# Complements dns-exposure (R-DNS-02, the EXTERNAL/public-record side). Together they cover the +# split-horizon pair: external answer = public record (R-DNS-02); internal answer = Gateway VIP +# (R-DNS-01/03). +# +# CLEANUP: chainsaw deletes the per-test namespace (Job included) at teardown. Read-only otherwise. +# chainsaw v0.2.12: Job + status assert; no test-level finally. +# ============================================================================= +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: dns-internal-resolution + labels: + conformance.home.k3s/R-DNS-01: "true" + conformance.home.k3s/R-DNS-03: "true" + annotations: + conformance.home.k3s/requirements: "R-DNS-01,R-DNS-03" + conformance.home.k3s/layer: "L2" + conformance.home.k3s/safety-critical: "false" + conformance.home.k3s/executable: "true" + conformance.home.k3s/intent: >- + An in-cluster pod resolving keycloak.home.keithjackson.dev via cluster DNS gets the internal + Gateway VIP (10.73.0.24x), not the public hairpin — split-horizon in-cluster resolution. +spec: + timeouts: + apply: 60s + assert: 2m + exec: 2m + steps: + - name: then-cluster-dns-answers-with-the-internal-gateway-vip + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: dns-internal-probe + spec: + backoffLimit: 2 + activeDeadlineSeconds: 100 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36.1 + command: + - sh + - -c + - | + set -u + host="keycloak.home.keithjackson.dev" + # Resolve via the DEFAULT cluster resolver (CoreDNS, /etc/resolv.conf -> + # 10.43.0.10). Split-horizon must answer with the in-cluster Gateway VIP + # (L2 pool 10.73.0.24x), proving pods hairpin through the Gateway. + out="$(nslookup "$host" 2>/dev/null)" + echo "$out" + ip="$(echo "$out" | awk '/^Address: /{print $2}' | grep -E '^10\.73\.0\.24[0-9]$' | head -1)" + if [ -n "$ip" ]; then + echo "INTERNAL_RESOLUTION_OK $host -> $ip (in-cluster Gateway VIP)" + exit 0 + fi + echo "FAIL: $host did not resolve to the in-cluster Gateway VIP (10.73.0.24x);" + echo " split-horizon coredns-custom not in effect (got a public/off-cluster answer)." + exit 1 + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: dns-internal-probe + status: + succeeded: 1 diff --git a/platform/coredns/coredns-custom.yaml b/platform/coredns/coredns-custom.yaml new file mode 100644 index 0000000..189307e --- /dev/null +++ b/platform/coredns/coredns-custom.yaml @@ -0,0 +1,45 @@ +--- +# Split-horizon DNS for cluster pods (R-DNS-01 / R-DNS-03). +# +# PROBLEM: an in-cluster client resolving a platform hostname over its PUBLIC name +# (e.g. `keycloak.home.keithjackson.dev`) gets the off-cluster/public answer — it does NOT reach the +# in-cluster Gateway. That path does not serve the `*.home.keithjackson.dev` SNI, so TLS fails with +# `remote error: tls: unrecognized name` (oauth2-proxy OIDC discovery, Grafana generic_oauth, etc.). +# +# FIX (split-horizon, docs/09 "Split-horizon resolution"): inside the cluster, answer the whole +# `home.keithjackson.dev` zone with the L2-announced ingress VIP of the `shared` Gateway. Pod -> +# public-hostname traffic then hairpins through the Gateway with the correct SNI -> wildcard cert -> +# works. External/LAN clients keep their own answers (PiHole split-horizon, docs/09) — this ConfigMap +# only affects CLUSTER DNS (CoreDNS), not LAN DNS. +# +# MECHANISM (k3s-specific, F-DNS / docs/09): k3s CoreDNS ships a Corefile with +# `import /etc/coredns/custom/*.server`, and mounts the `coredns-custom` ConfigMap (kube-system) at +# /etc/coredns/custom. Each `*.server` key becomes a top-level server block imported into the +# Corefile. CoreDNS `reload` picks up ConfigMap changes automatically — NO pod restart needed +# (GitOps-friendly; the change reconciles live). +# +# ${GATEWAY_VIP} is substituted by Flux postBuild.substituteFrom cluster-vars (R-ENV-02): dev = the +# dev L2 pool VIP, prod = the prod LAN VIP. The unified graph is identical across environments; only +# the cluster-var differs. See clusters/base/cluster-vars.yaml. +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + # Filename `home.server` -> imported as a standalone server block for the whole zone. + home.server: | + home.keithjackson.dev:53 { + errors + # Answer EVERY name in the zone (apex + all `.home.keithjackson.dev`) with the Gateway + # VIP. The template plugin's default match is `.*`, so this covers current and FUTURE app + # hostnames without editing this file per service. `{{ .Name }}` echoes the queried FQDN. + template IN A { + answer "{{ .Name }} 60 IN A ${GATEWAY_VIP}" + } + # AAAA -> NODATA (NOERROR, no answer): the name has an A but no AAAA, so IPv6-capable clients + # correctly fall back to the A record instead of getting NXDOMAIN/SERVFAIL. + template IN AAAA { + rcode NOERROR + } + } diff --git a/platform/coredns/kustomization.yaml b/platform/coredns/kustomization.yaml new file mode 100644 index 0000000..2de8301 --- /dev/null +++ b/platform/coredns/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# platform/coredns — in-cluster split-horizon DNS (R-DNS-01 / R-DNS-03). +# A single `coredns-custom` ConfigMap in kube-system that k3s CoreDNS imports as an extra server +# block (see coredns-custom.yaml). Rendered by the `platform` Flux Kustomization, which supplies +# ${GATEWAY_VIP} via substituteFrom cluster-vars (R-ENV-02). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - coredns-custom.yaml diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index 0a85f58..f889423 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -17,6 +17,7 @@ resources: - cert-manager # Let's Encrypt DNS-01 wildcard — needs SOPS `cloudflare-api-token` - externaldns # auto DNS records — needs SOPS `cloudflare-api-token` - gateway # Gateway API listeners (ingress; replaces Traefik) + - coredns # split-horizon in-cluster DNS: *.home.keithjackson.dev -> Gateway VIP (R-DNS-01/03) # ---- Phase 3 — Egress / VPN (ADR-0015): node-level host OpenVPN split-tunnel (Ansible role # vpn_egress_gateway) steered by a Cilium Egress Gateway policy, with a host nftables kill-switch. # No in-cluster gateway pod (Gluetun retired). See ADR-0015 + DECISIONS-LOG "Phase 3 VPN". ----