fix(conformance): workload-mTLS certificates may use a private CA, if they prove they are workload-internal

R-TLS-01/docs/08 govern externally reachable TLS and ADR-0006 puts workload mTLS out of scope, but
tls-lifecycle-automated required ACME for every Certificate, which platform/buildkit's internal CA
cannot satisfy. Clause (5b) accepts the tls-scope=workload-mtls annotation only when the
certificate proves it: no Gateway serves it, internal names only, namespaced selfSigned/ca issuer
backed by an exempt CA certificate. Also fixes yamllint comma spacing in the buildkitd mounts.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

conformance/tests/l0-static/tls-lifecycle-automated/check.sh

@@ -42,6 +42,19 @@
 #           whose every solver is an automated challenge type (dns01 / http01) — the
 #           same code path serves issuance AND renewal, so a solver a human must
 #           complete would defeat both;
+#       (5b) the ONE exception to (5) — WORKLOAD mTLS, added 2026-10-29 — is a Certificate
+#           annotated `homelab.keithjackson.dev/tls-scope: workload-mtls`. The spec's TLS
+#           requirements are about EXTERNALLY REACHABLE services (R-TLS-01; docs/08), and
+#           ADR-0006 says outright that "mTLS between workloads is a separate concern … not
+#           covered here". A public ACME CA cannot issue for a `*.svc.cluster.local` name or a
+#           client-auth certificate at all. The annotation is NOT trusted: an exempt
+#           Certificate must PROVE it is workload-internal — its Secret is served by NO Gateway
+#           listener, every DNS name / commonName is cluster-internal (a bare label or
+#           `*.svc[.cluster.local]`), it carries no IP/URI/email SANs, and its issuer is a
+#           NAMESPACED Issuer in its own namespace whose spec is `selfSigned` or a `ca` backed
+#           by a Secret that a tracked, likewise-exempt Certificate produces. Issuance and
+#           renewal stay unattended (cert-manager renews CA-issued certs itself), and (9)
+#           still applies. Every exemption is printed. First user: platform/buildkit.
 #       (6) each dns01 solver's provider credential is referenced BY NAME and that
 #           Secret is delivered SOPS-encrypted from git via a kustomization — the
 #           operator's one-time token hand-off (docs/08 OQ-4), never a per-cert step;
@@ -265,6 +278,7 @@ CERT_SECRETS: dict[tuple[str, str], str] = {           # (namespace, secretName)
     for rel, d in CERTS
 }
 listeners_checked = 0
+GATEWAY_SECRETS: set[tuple[str, str]] = set()          # every (namespace, Secret) a listener serves
 for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
     gw_ns = ns_of(gw)
     for lis in ((gw.get("spec") or {}).get("listeners") or []):
@@ -280,6 +294,7 @@ for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
         for r in refs:
             kind = str(r.get("kind", "Secret"))
             key = (str(r.get("namespace") or gw_ns), str(r.get("name")))
+            GATEWAY_SECRETS.add(key)
             check(kind == "Secret",
                   f"{rel}: listener {lis.get('name')!r} certificateRef kind is {kind!r}; only "
                   "Secret refs can be traced to a cert-manager Certificate")
@@ -324,8 +339,61 @@ CM_NS = ns_of(cm_releases[0][1], "cert-manager") if cm_releases else "cert-manag
 ISSUER_INDEX = {(d.get("kind"), name_of(d)): (rel, d) for rel, d in ISSUERS}
 solver_creds: list[tuple[str, str, str, str]] = []     # (issuer, ns, secretName, ref field)
 issuers_verified: set[str] = set()
+
+# (5b) workload-mTLS exemption — see the header. Proven, never taken on trust.
+SCOPE_ANN = "homelab.keithjackson.dev/tls-scope"
+INTERNAL_NAME = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$")
+NS_ISSUERS = {(ns_of(d), name_of(d)): (rel, d) for rel, d in of_kind("Issuer", "cert-manager.io/")}
+EXEMPT_SECRETS = {(ns_of(d), str((d.get("spec") or {}).get("secretName", "")))
+                  for _, d in CERTS
+                  if ((d.get("metadata") or {}).get("annotations") or {}).get(SCOPE_ANN) == "workload-mtls"}
+exempted: list[str] = []
+
+def workload_mtls_ok(rel: str, cert: dict) -> bool:
+    """True if `cert` claims the workload-mTLS scope AND proves it; records failures."""
+    ann = ((cert.get("metadata") or {}).get("annotations") or {}).get(SCOPE_ANN)
+    if ann is None:
+        return False
+    cns, cname = ns_of(cert), name_of(cert)
+    spec = cert.get("spec") or {}
+    ok = True
+    def need(cond, msg):
+        nonlocal ok
+        check(cond, f"{rel}: Certificate {cns}/{cname} claims {SCOPE_ANN}={ann!r} but {msg}")
+        ok = ok and bool(cond)
+    need(ann == "workload-mtls", "the only accepted value is 'workload-mtls'")
+    need((cns, str(spec.get("secretName", ""))) not in GATEWAY_SECRETS,
+         "its Secret is served by a Gateway listener — that is external TLS, which must be ACME (R-TLS-01)")
+    names = [str(n) for n in (spec.get("dnsNames") or [])]
+    if spec.get("commonName"):
+        names.append(str(spec["commonName"]))
+    for n in names:
+        internal = INTERNAL_NAME.match(n) and ("." not in n or n.endswith(".svc") or n.endswith(".svc.cluster.local"))
+        need(bool(internal), f"name {n!r} is not cluster-internal (bare label or *.svc[.cluster.local])")
+    for f in ("ipAddresses", "uris", "emailAddresses"):
+        need(not spec.get(f), f"it carries {f}, which a workload-internal certificate has no use for")
+    ref = spec.get("issuerRef") or {}
+    need(str(ref.get("kind", "Issuer")) == "Issuer",
+         "its issuer is not a NAMESPACED Issuer — a ClusterIssuer could sign for any namespace")
+    hit = NS_ISSUERS.get((cns, str(ref.get("name", ""))))
+    need(hit is not None, f"its Issuer {ref.get('name')!r} is not tracked in namespace {cns!r}")
+    if hit is not None:
+        ispec = hit[1].get("spec") or {}
+        kinds = sorted(ispec)
+        need(kinds in (["selfSigned"], ["ca"]),
+             f"its Issuer is {kinds} — only selfSigned or ca are accepted for workload mTLS")
+        if kinds == ["ca"]:
+            need((cns, str((ispec.get("ca") or {}).get("secretName", ""))) in EXEMPT_SECRETS,
+                 "its CA Issuer's Secret is not produced by a tracked workload-mtls Certificate "
+                 "in the same namespace — the CA itself would have to be minted by hand")
+    if ok:
+        exempted.append(f"{cns}/{cname}")
+    return ok
+
 for rel, cert in CERTS:
     spec = cert.get("spec") or {}
+    if workload_mtls_ok(rel, cert):
+        continue          # (5b): proven workload-internal; (9) below still applies
     ref = spec.get("issuerRef") or {}
     kind = str(ref.get("kind", "Issuer"))
     per_env = resolve_per_env(str(ref.get("name", "")))
@@ -379,6 +447,8 @@ for rel, cert in CERTS:
                             solver_creds.append((iname, want_ns, n, f"{provider}.{field}"))
 
 check(len(issuers_verified) > 0, "no issuer was reachable from any Certificate — (5) proved nothing")
+if exempted:
+    print(f"  (5b) workload-mTLS certificates exempted from ACME after proving it: {', '.join(exempted)}")
 
 # (6) the solver credential is delivered, SOPS-encrypted, from git — never per-cert by hand.
 check(len(solver_creds) > 0,

decisions/2026-10-29-image-pipeline.md

@@ -85,6 +85,23 @@ valid_volumes) is **untouched**.
   - AppArmor enabled (`Y`).
 
   So this is a satisfied prerequisite, not an open one.
+- **The TLS conformance check needed a narrow, self-proving exception** (`conformance/**`, so this
+  changes what green means; flagged for the reviewer).
+  - The first gate run failed L0 `tls-lifecycle-automated` clause (5): "every Certificate's
+    issuer must be ACME". That is broader than the spec it serves:
+    - R-TLS-01 and docs/08 concern *externally reachable* services;
+    - ADR-0006 says "mTLS between workloads is a separate concern … not covered here";
+    - a public CA cannot issue for a `*.svc.cluster.local` name or a client-auth certificate.
+  - The new clause (5b) exempts a Certificate annotated
+    `homelab.keithjackson.dev/tls-scope: workload-mtls` **only if it proves it**:
+    - no Gateway listener serves its Secret;
+    - every DNS name and the commonName are a bare label or `*.svc[.cluster.local]`;
+    - it has no IP, URI or email SANs;
+    - its issuer is a namespaced `selfSigned`/`ca` Issuer in its own namespace;
+    - a `ca` issuer's Secret is produced by a likewise-exempt tracked Certificate.
+  - Exemptions are printed, and clause (9), the renewal arithmetic, still applies.
+  - **Negative-tested:** a public DNS name, and a Gateway serving `buildkitd-tls`, each fail the
+    check with a specific message.
 - **`infra/gitea/buildkit-client-secret.sh`** (operator action): copies `ca.crt`, `tls.crt` and
   `tls.key` from the Secret `buildkit/buildkit-client-ci` into the Actions secrets `BUILDKIT_CA`,
   `BUILDKIT_CLIENT_CERT` and `BUILDKIT_CLIENT_KEY`. Before writing anything it checks that:

platform/buildkit/certificates.yaml

@@ -14,6 +14,15 @@
 # tls.key from the Secret below into the repository secrets BUILDKIT_CA, BUILDKIT_CLIENT_CERT and
 # BUILDKIT_CLIENT_KEY without printing them. Re-run it after every renewal (see renewBefore).
 #
+# WHY A PRIVATE CA AND NOT LET'S ENCRYPT. The platform's TLS rule (R-TLS-01/02, docs/08,
+# ADR-0006: ACME DNS-01) is about EXTERNALLY REACHABLE services; ADR-0006 itself says "mTLS between
+# workloads is a separate concern … not covered here", and a public CA cannot issue for a
+# `*.svc.cluster.local` name or a client-auth certificate anyway. buildkitd is ClusterIP-only. Each
+# Certificate below is therefore annotated `homelab.keithjackson.dev/tls-scope: workload-mtls`,
+# which the L0 check tls-lifecycle-automated accepts ONLY if the certificate proves it: no
+# Gateway serves its Secret, every name is cluster-internal, and the issuer is a namespaced
+# selfSigned/CA Issuer whose CA is itself issued here. Issuance and renewal stay unattended.
+#
 # ROTATION. cert-manager renews both leaves `renewBefore` their expiry and rewrites the Secrets.
 #   * server: buildkitd reads its certificate at START, so the pod must restart inside the 30-day
 #     renewal window (a node reboot by kured, a Deployment rollout, or a manual restart).
@@ -32,6 +41,8 @@ kind: Certificate
 metadata:
   name: buildkit-ca
   namespace: buildkit
+  annotations:
+    homelab.keithjackson.dev/tls-scope: workload-mtls   # see the header: R-TLS-01 is about external TLS
 spec:
   isCA: true
   commonName: buildkit-ca
@@ -58,6 +69,8 @@ kind: Certificate
 metadata:
   name: buildkitd-tls
   namespace: buildkit
+  annotations:
+    homelab.keithjackson.dev/tls-scope: workload-mtls
 spec:
   secretName: buildkitd-tls
   commonName: buildkitd.buildkit.svc.cluster.local
@@ -83,6 +96,8 @@ kind: Certificate
 metadata:
   name: buildkit-client-ci
   namespace: buildkit
+  annotations:
+    homelab.keithjackson.dev/tls-scope: workload-mtls
 spec:
   secretName: buildkit-client-ci
   commonName: gitea-actions-image-pipeline

platform/buildkit/deployment.yaml

@@ -114,13 +114,13 @@ spec:
             limits:
               memory: 4Gi
           volumeMounts:
-            - { name: certs,   mountPath: /certs, readOnly: true }
-            - { name: config,  mountPath: /etc/buildkit, readOnly: true }
-            - { name: cache,   mountPath: /home/user/.local/share/buildkit }
-            - { name: run,     mountPath: /run/user }
-            - { name: tmpdir,  mountPath: /home/user/.local/tmp }   # the image's TMPDIR
-            - { name: tmp,     mountPath: /tmp }
-            - { name: clicfg,  mountPath: /home/user/.docker }      # buildctl (the probes) writes here
+            - { name: certs, mountPath: /certs, readOnly: true }
+            - { name: config, mountPath: /etc/buildkit, readOnly: true }
+            - { name: cache, mountPath: /home/user/.local/share/buildkit }
+            - { name: run, mountPath: /run/user }
+            - { name: tmpdir, mountPath: /home/user/.local/tmp }   # the image's TMPDIR
+            - { name: tmp, mountPath: /tmp }
+            - { name: clicfg, mountPath: /home/user/.docker }      # buildctl (the probes) writes here
       volumes:
         - name: certs
           secret: