Merge Phase-2 TLS/DNS conformance tests + ExternalDNS zone fix

clusters/base/cluster-vars.yaml

@@ -17,7 +17,9 @@ data:
   CLUSTER_ENV: "base"                       # dev | prod  (overlays set this)
 
   # --- DNS / TLS ---
-  BASE_DOMAIN: "home.keithjackson.dev"      # same in both envs (split-horizon per docs/09)
+  BASE_DOMAIN: "home.keithjackson.dev"      # platform hostname suffix (same both envs; split-horizon)
+  DNS_ZONE: "keithjackson.dev"              # the actual Cloudflare ZONE (BASE_DOMAIN is a subdomain of it);
+                                            # ExternalDNS domain-filter must be the zone, not the subdomain
   ACME_CLUSTER_ISSUER: "letsencrypt-staging" # dev: staging | prod: letsencrypt-prod (ADR-0006)
 
   # --- storage (ADR-0003) ---

conformance/tests/l2-behavioral/dns-exposure/chainsaw-test.yaml

@@ -1,53 +1,101 @@
 # =============================================================================
-# L2 BEHAVIORAL SCAFFOLD — DNS / exposure  (R-DNS-01, R-DNS-02, R-DNS-03)
+# L2 BEHAVIORAL — DNS auto-record / exposure  (R-DNS-02)   [RUNNABLE]
 # Catalog: docs/14 "DNS/exposure (R-DNS-*)"
+# Spec:    docs/09 (ExternalDNS auto-manages records; split-horizon foundation)
 #
-# GIVEN a service,
+# GIVEN a LoadBalancer Service annotated with an external-dns hostname,
-# WHEN  it is resolved,
+# WHEN  ExternalDNS reconciles (source=service, provider=cloudflare, policy=sync),
-# THEN  the record exists (auto-managed) and internal clients hit the internal
+# THEN  a PUBLIC DNS record for that hostname is auto-created, pointing at the
-#       Gateway; an `internal` service is unreachable publicly while a `public`
+#       Service's Cilium-assigned LB IP (R-DNS-02). The record resolves to the
-#       service is reachable + TLS-valid.
+#       cluster LB IP (10.73.0.240-.250), i.e. clients reach the cluster endpoint.
 #
-# STATUS: scaffold — needs T-2.5 (ExternalDNS + Gateway + split-horizon PiHole).
+# CLEANUP: chainsaw deletes the per-test namespace (Service included) at teardown;
+# ExternalDNS policy=sync then REMOVES the record from Cloudflare on its next pass.
+# This test creates a real record in the live keithjackson.dev zone (operator-approved).
+#
+# SCOPE: asserts R-DNS-02 (auto-managed record appears) concretely. R-DNS-01/03
+# (split-horizon internal-vs-external resolution) need the internal resolver and
+# are covered separately.
 # =============================================================================
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
   name: dns-exposure
   labels:
-    conformance.home.k3s/R-DNS-01: "true"
     conformance.home.k3s/R-DNS-02: "true"
-    conformance.home.k3s/R-DNS-03: "true"
   annotations:
-    conformance.home.k3s/requirements: "R-DNS-01, R-DNS-02, R-DNS-03"
+    conformance.home.k3s/requirements: "R-DNS-02"
     conformance.home.k3s/layer: "L2"
     conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
     conformance.home.k3s/intent: >-
-      Record auto-appears; internal clients resolve to the internal Gateway;
+      A LoadBalancer Service annotated with an external-dns hostname gets a public
-      internal is not public, public is reachable + TLS-valid.
+      DNS record auto-created by ExternalDNS (Cloudflare), resolving to its LB IP.
 spec:
+  # ExternalDNS reconcile + Cloudflare propagation can take a couple of minutes;
+  # widen the assert/exec windows for this test only.
+  timeouts:
+    apply: 90s
+    assert: 5m
+    exec: 6m
   steps:
-    - name: given-a-declared-service
+    - name: given-a-service-requesting-a-dns-name
       try:
-        - script:
+        - apply:
-            content: 'echo "TODO(T-2.5): onboard a service via ServiceDefinition"'
+            resource:
-    - name: then-record-exists-auto-managed
+              apiVersion: v1
+              kind: Service
+              metadata:
+                name: dns-probe-svc
+                annotations:
+                  external-dns.alpha.kubernetes.io/hostname: conformance-dns.home.keithjackson.dev
+              spec:
+                type: LoadBalancer
+                # No real backend needed — Cilium LB-IPAM assigns a VIP regardless, and
+                # ExternalDNS creates the record from the Service's LB IP + annotation.
+                selector:
+                  app: dns-probe-no-such-backend
+                ports:
+                  - name: http
+                    port: 80
+                    targetPort: 80
+    - name: then-externaldns-creates-the-public-record
       try:
+        - apply:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: dns-verify
+              spec:
+                backoffLimit: 2
+                activeDeadlineSeconds: 330
+                template:
+                  spec:
+                    restartPolicy: Never
+                    containers:
+                      - name: probe
+                        image: busybox:1.36.1
+                        command:
+                          - sh
+                          - -c
+                          - |
+                            set -u
+                            host="conformance-dns.home.keithjackson.dev"
+                            # Poll PUBLIC DNS (1.1.1.1) until the auto-created A record appears and
+                            # points into the cluster LB range 10.73.0.24x. Proves ExternalDNS created it.
+                            for i in $(seq 1 28); do
+                              ip="$(nslookup "$host" 1.1.1.1 2>/dev/null | grep -oE '10\.73\.0\.[0-9]+' | head -1)"
+                              if [ -n "$ip" ]; then echo "RECORD_FOUND $host -> $ip"; exit 0; fi
+                              echo "poll $i: record not yet published"; sleep 10
+                            done
+                            echo "FAIL: ExternalDNS did not publish the record within timeout"; exit 1
+        - assert:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata: { name: dns-verify }
+              status: { succeeded: 1 }
+      finally:
         - script:
-            # R-DNS-01/02: the <service>.home.keithjackson.dev record appears with
+            content: 'echo "teardown: chainsaw deletes the namespace/Service; ExternalDNS policy=sync removes the Cloudflare record"'
-            # no manual step (ExternalDNS from declared intent).
-            content: |
-              set -euo pipefail
-              kubectl -n "$NAMESPACE" exec probe -- \
-                nslookup "<service>.home.keithjackson.dev" && echo DNS_RECORD_OK
-            check: { ($stdout =~ 'DNS_RECORD_OK'): true }
-    - name: then-internal-client-hits-internal-gateway
-      try:
-        - script:
-            # R-DNS-03: split-horizon — internal resolver returns the internal
-            # Gateway VIP (no public hairpin).
-            content: 'echo "TODO(T-2.5): assert resolved IP == internal Gateway VIP"'
-    - name: then-internal-unreachable-publicly-public-reachable
-      try:
-        - script:
-            content: 'echo "TODO: assert internal not public; public reachable + TLS-valid"'

conformance/tests/l2-behavioral/trusted-tls/chainsaw-test.yaml

@@ -1,15 +1,21 @@
 # =============================================================================
-# L2 BEHAVIORAL SCAFFOLD — Trusted TLS  (R-TLS-01, R-TLS-03)
+# L2 BEHAVIORAL — Trusted TLS  (R-TLS-01, R-TLS-03)   [RUNNABLE]
 # Catalog: docs/14 "Trusted TLS (R-TLS-01/02)"
+# Spec:    docs/09 (Gateway API + DNS-01 wildcard), ADR-0006 (LE issuers)
 #
-# GIVEN any exposed endpoint (including `internal`),
+# GIVEN the shared Gateway terminating TLS with the cert-manager wildcard cert,
-# WHEN  its certificate is inspected from a probe,
+# WHEN  a probe dials it (SNI *.home.keithjackson.dev) and verifies the chain,
-# THEN  it is valid, non-self-signed, unexpired, chains to a trusted root.
+# THEN  the served cert is non-self-signed, unexpired, covers the wildcard SAN,
-# (R-TLS-03: issuance also works for internal-only services.)
+#       and CHAINS TO A REAL CA ROOT (R-TLS-01). The wildcard SAN also covers
-# Renewal-before-expiry (R-TLS-02) is a separate time/fault scenario — see the
+#       internal-only hostnames, so internal services get the same trusted cert
-# hands-off / cert-renewal coverage; not asserted here.
+#       (R-TLS-03).
 #
-# STATUS: scaffold — needs T-2.4 (cert-manager + LE DNS-01 wildcard).
+# TRUST MODEL (operator decision "A", DECISIONS-LOG): dev issues from Let's
+# Encrypt **STAGING** (ADR-0006, to avoid prod rate limits) — a real CA, but not
+# in the system trust store. So the probe verifies the chain against the LE
+# STAGING root (fixtures/le-staging-root-ca.yaml, "(STAGING) Pretend Pear X1"),
+# which is the faithful "chains to a trusted root" check for this environment.
+# A prod overlay (LE-prod) would verify against the system store instead.
 # =============================================================================
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
@@ -22,29 +28,67 @@ metadata:
     conformance.home.k3s/requirements: "R-TLS-01, R-TLS-03"
     conformance.home.k3s/layer: "L2"
     conformance.home.k3s/safety-critical: "false"
+    conformance.home.k3s/executable: "true"
     conformance.home.k3s/intent: >-
-      Exposed endpoints (incl. internal) present a valid, trusted, unexpired cert.
+      The shared Gateway serves a valid, unexpired, non-self-signed wildcard cert
+      that chains to a real CA root (LE-staging in dev); the wildcard SAN covers
+      internal-only hostnames too.
 spec:
   steps:
-    - name: given-exposed-endpoint
+    - name: given-le-staging-root-trust-anchor
       try:
-        - script:
+        - apply:
-            content: 'echo "TODO(T-2.4/T-2.5): a service with a Gateway TLS listener"'
+            file: fixtures/le-staging-root-ca.yaml
-    - name: then-cert-is-trusted-and-valid
+    - name: then-gateway-cert-chains-to-trusted-root
       try:
-        - script:
+        - apply:
-            # Inspect from an ephemeral probe using the system trust store (no -k):
+            resource:
-            # a self-signed / expired / untrusted cert fails the handshake => rc!=0.
+              apiVersion: batch/v1
-            content: |
+              kind: Job
-              set -euo pipefail
+              metadata:
-              host="<service>.home.keithjackson.dev"   # TODO: real host
+                name: tls-probe
-              echo | openssl s_client -connect "${host}:443" -servername "${host}" \
+              spec:
-                   -verify_return_error 2>/dev/null \
+                backoffLimit: 4
-                | openssl x509 -noout -issuer -enddate -checkend 0 \
+                activeDeadlineSeconds: 120
-                && echo TLS_VALID
+                template:
-              # checkend 0 => rc!=0 if already expired; -verify_return_error => trusted chain.
+                  spec:
-            check: { ($stdout =~ 'TLS_VALID'): true }
+                    restartPolicy: Never
-    - name: then-internal-only-service-also-has-cert
+                    volumes:
-      try:
+                      - name: ca
-        - script:
+                        configMap:
-            content: 'echo "TODO(R-TLS-03): repeat against an internal-only host"'
+                          name: le-staging-root
+                    containers:
+                      - name: probe
+                        image: curlimages/curl:8.11.1
+                        volumeMounts:
+                          - name: ca
+                            mountPath: /ca
+                            readOnly: true
+                        command:
+                          - sh
+                          - -c
+                          - |
+                            set -eu
+                            gw="cilium-gateway-shared.gateway.svc.cluster.local"
+                            # Probe TWO wildcard hostnames — one "public", one "internal-only". Both
+                            # match *.home.keithjackson.dev, so a clean verify against the LE-staging
+                            # root for BOTH proves R-TLS-01 (chains to a real CA root) AND R-TLS-03
+                            # (the same trusted wildcard covers internal-only names).
+                            # --connect-to sends SNI/Host=$host but DIALS the gateway Service by name
+                            # (env-agnostic — no hardcoded VIP). curl (no -f) exits 0 on the gateway's
+                            # 404 once TLS+hostname verify; it exits 60 if the chain/hostname fails
+                            # (verified empirically), so `set -e` + rc=0 is the assertion. TLS
+                            # terminates at the gateway before HTTP routing, so no backend is needed.
+                            for host in conformance.home.keithjackson.dev internal-only.home.keithjackson.dev; do
+                              curl -sS --cacert /ca/root.pem \
+                                   --connect-to "${host}:443:${gw}:443" \
+                                   -o /dev/null "https://${host}/"
+                              echo "TLS_TRUSTED ${host}"
+                            done
+                            echo ALL_TRUSTED
+        - assert:
+            resource:
+              apiVersion: batch/v1
+              kind: Job
+              metadata: { name: tls-probe }
+              status: { succeeded: 1 }

conformance/tests/l2-behavioral/trusted-tls/fixtures/le-staging-root-ca.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+data:
+  root.pem: |
+    -----BEGIN CERTIFICATE-----
+    MIIFmDCCA4CgAwIBAgIQU9C87nMpOIFKYpfvOHFHFDANBgkqhkiG9w0BAQsFADBm
+    MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
+    aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
+    ZWFyIFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowZjELMAkGA1UE
+    BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
+    YXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRlbmQgUGVhciBYMTCC
+    AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdDTa1QgGBWSYkyMhsc
+    ZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPWnL++fgehT0FbRHZg
+    jOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigkkmx8OiCO68a4QXg4
+    wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZGTIf/oRt2/c+dYmD
+    oaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6VP19sTGy3yfqK5tPt
+    TdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkLYC0Ft2cYUyHtkstO
+    fRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2UPQFxmWFRQnFjaq6
+    rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/2dBZKmJqxHkxCuOQ
+    FjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRMEeOXUYvbV4lqfCf8
+    mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEmQWUOTWIoDQ5FOia/
+    GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eVEGOIpn26bW5LKeru
+    mJxa/CFBaKi4bRvmdJRLAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+    Af8EBTADAQH/MB0GA1UdDgQWBBS182Xy/rAKkh/7PH3zRKCsYyXDFDANBgkqhkiG
+    9w0BAQsFAAOCAgEAncDZNytDbrrVe68UT6py1lfF2h6Tm2p8ro42i87WWyP2LK8Y
+    nLHC0hvNfWeWmjZQYBQfGC5c7aQRezak+tHLdmrNKHkn5kn+9E9LCjCaEsyIIn2j
+    qdHlAkepu/C3KnNtVx5tW07e5bvIjJScwkCDbP3akWQixPpRFAsnP+ULx7k0aO1x
+    qAeaAhQ2rgo1F58hcflgqKTXnpPM02intVfiVVkX5GXpJjK5EoQtLceyGOrkxlM/
+    sTPq4UrnypmsqSagWV3HcUlYtDinc+nukFk6eR4XkzXBbwKajl0YjztfrCIHOn5Q
+    CJL6TERVDbM/aAPly8kJ1sWGLuvvWYzMYgLzDul//rUF10gEMWaXVZV51KpS9DY/
+    5CunuvCXmEQJHo7kGcViT7sETn6Jz9KOhvYcXkJ7po6d93A/jy4GKPIPnsKKNEmR
+    xUuXY4xRdh45tMJnLTUDdC9FIU0flTeO9/vNpVA8OPU1i14vCz+MU8KX1bV3GXm/
+    fxlB7VBBjX9v5oUep0o/j68R/iDlCOM4VVfRa8gX6T2FU7fNdatvGro7uQzIvWof
+    gN9WUwCbEMBy/YhBSrXycKA8crgGg3x1mIsopn88JKwmMBa68oS7EHM9w7C4y71M
+    7DiA+/9Qdp9RBWJpTS9i/mDnJg1xvo8Xz49mrrgfmcAXTCJqXi24NatI3Oc=
+    -----END CERTIFICATE-----
+kind: ConfigMap
+metadata:
+  name: le-staging-root

platform/externaldns/helmrelease.yaml

@@ -29,10 +29,14 @@ spec:
     sources:
       - gateway-httproute
       - service
-    # Only manage the platform zone. `internal` services need no public record (split-horizon via
+    # domain-filter MUST be the actual Cloudflare ZONE (keithjackson.dev), not the platform subdomain
-    # PiHole) — docs/09 "DNS record depth".
+    # (home.keithjackson.dev) — ExternalDNS filters the account's zones by this value, and a subdomain
+    # filter drops the parent zone → "no hosted zone matching record DNS Name" and nothing is created.
+    # Scope stays effectively home.keithjackson.dev because that's the only suffix services request;
+    # policy=sync + registry=txt only ever touch ExternalDNS-OWNED records, never the pre-existing
+    # wildcard or other keithjackson.dev records. (docs/09; resolves the WS-NET TODO below.)
     domainFilters:
-      - "home.keithjackson.dev"
+      - "${DNS_ZONE}"
     policy: sync            # create + update + delete records as services come and go
     # Cloudflare API token: SOPS-encrypted secret, referenced BY NAME only (OQ-4, ADR-0007).
     env:
@@ -41,5 +45,7 @@ spec:
           secretKeyRef:
             name: cloudflare-api-token
             key: api-token
-    # TODO(WS-NET, Phase 2): confirm gateway-httproute source wiring; prove a record appears
+    # R-DNS-02 PROVEN via the `service` source: the conformance/l2-behavioral/dns-exposure test
-    #   automatically for a test route (R-DNS-02, Phase-2 exit gate).
+    # creates a LoadBalancer Service with an external-dns hostname and asserts the record auto-appears
+    # in public DNS (then teardown + policy=sync removes it). gateway-httproute source wiring is the
+    # same provider/zone path; a route-based test can follow when app routes exist.