Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5.5
|
9c51f60a35 | ||
|
jakman85andClaude Opus 5.5
|
267446c68f | ||
|
jakman85
|
d1994b16e8 | ||
 renovate-bot
|
1a94b0535c |
No files matched your search
@@ -26,7 +26,10 @@
|
||||
# KNOWN_NON_CATALOG_PUBLISHERS — shrink-only, stale entries FAIL;
|
||||
# (7) the check is LOAD-BEARING: whole-repo scope declared in chainsaw-test.yaml, honoured by
|
||||
# ci/l0.sh's --include-scope union, and ci/gate.sh blocks on an l0 failure. NO hand-written
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears).
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears);
|
||||
# (8) the external-dns HelmRelease declares `annotationPrefix` explicitly and it matches the
|
||||
# prefix of EVERY external-dns annotation in the tree and in catalog/render.py — added after
|
||||
# the 2026-11-07 v0.23.0 default-prefix change silently voided (1)-(3) at runtime.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — it is STATIC, offline, cluster-free):
|
||||
# - That the record is gone from Cloudflare. Live read; belongs beside
|
||||
@@ -90,9 +93,18 @@ CHECKS = 0
|
||||
# ControllerValue = "dns-controller" -> the value that CLAIMS an object
|
||||
# source/annotations/processors.go IsControllerMismatch skips a source object when the key is
|
||||
# PRESENT and the value is NOT ControllerValue. So "disowned" == key present, value != claimed.
|
||||
CONTROLLER_KEY = "external-dns.alpha.kubernetes.io/controller"
|
||||
#
|
||||
# THE PREFIX IS NOT A CONSTANT OF NATURE. Every key external-dns reads is `<annotationPrefix><name>`,
|
||||
# and the prefix is a runtime flag whose DEFAULT moved in v0.22.0 (#6424: alpha -> GA, "with no
|
||||
# fallback"). On 2026-11-07 the chart bump to v0.23.0 made every key below invisible to the running
|
||||
# controller while this check — which only asserted the keys were PRESENT — stayed green. (8) now
|
||||
# ties ANNOTATION_PREFIX to the HelmRelease's explicit `annotationPrefix`.
|
||||
ANNOTATION_PREFIX = "external-dns.alpha.kubernetes.io/"
|
||||
CONTROLLER_KEY = ANNOTATION_PREFIX + "controller"
|
||||
CONTROLLER_CLAIMED = "dns-controller"
|
||||
HOSTNAME_KEY = "external-dns.alpha.kubernetes.io/hostname"
|
||||
HOSTNAME_KEY = ANNOTATION_PREFIX + "hostname"
|
||||
# Any key that looks like an external-dns annotation, under ANY prefix variant (alpha, GA, typos).
|
||||
EXTERNAL_DNS_KEY_RE = re.compile(r"^external-dns(\.[a-z0-9-]+)*\.kubernetes\.io/")
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
@@ -231,9 +243,7 @@ for sub in SCAN_DIRS:
|
||||
ann = md.get("annotations") or {}
|
||||
if kind == "HTTPRoute":
|
||||
routes.append((rel, d))
|
||||
elif kind == "Service" and any(
|
||||
a.startswith("external-dns.alpha.kubernetes.io/") for a in ann
|
||||
):
|
||||
elif kind == "Service" and any(EXTERNAL_DNS_KEY_RE.match(a) for a in ann):
|
||||
dns_services.append((rel, d))
|
||||
|
||||
check(len(routes) > 0, "inventory: no HTTPRoute found under apps/ platform/ runbooks/ — parse "
|
||||
@@ -444,6 +454,67 @@ check(
|
||||
"and the whole-repo-invariants-wired meta-test fails the gate on a hand-written stage.",
|
||||
)
|
||||
|
||||
# =============================================================================================
|
||||
# (8) THE RUNNING CONTROLLER READS THE PREFIX THIS TREE WRITES.
|
||||
# (a) the external-dns HelmRelease sets `values.annotationPrefix` EXPLICITLY (an upstream
|
||||
# default is exactly what moved under us) and it equals ANNOTATION_PREFIX;
|
||||
# (b) it does not ALSO enable `--enable-legacy-annotation-prefix` (dual-read hides a
|
||||
# half-migration; upstream says the flag will be removed);
|
||||
# (c) every external-dns-looking annotation key on any object under apps/ platform/ runbooks/
|
||||
# uses ANNOTATION_PREFIX — a key under another prefix is silently ignored at runtime;
|
||||
# (d) catalog/render.py emits no external-dns key under another prefix.
|
||||
# Migrating to the GA prefix = change ANNOTATION_PREFIX, the HelmRelease value, the renderer
|
||||
# and every manifest in ONE commit; this assertion fails on any partial state.
|
||||
# =============================================================================================
|
||||
HR_PATH = ROOT / "platform/externaldns/helmrelease.yaml"
|
||||
hr = next((d for d in docs_of(HR_PATH) if d.get("kind") == "HelmRelease"
|
||||
and (d.get("metadata") or {}).get("name") == "external-dns"), None)
|
||||
check(hr is not None, f"(8) {HR_PATH.relative_to(ROOT)} holds no HelmRelease `external-dns` — "
|
||||
f"cannot prove which annotation prefix the controller reads")
|
||||
values = ((hr or {}).get("spec") or {}).get("values") or {}
|
||||
hr_prefix = values.get("annotationPrefix")
|
||||
check(
|
||||
hr_prefix == ANNOTATION_PREFIX,
|
||||
f"(8a) external-dns HelmRelease `values.annotationPrefix` is {hr_prefix!r}, but every "
|
||||
f"external-dns annotation in this tree uses {ANNOTATION_PREFIX!r}. A mismatch (or relying on "
|
||||
f"the upstream default, which changed in v0.22.0) makes the controller IGNORE `controller: "
|
||||
f"none`, the Gateway `target` and Service `hostname` — internal names go public, public names "
|
||||
f"answer RFC1918 (incident 2026-11-07).",
|
||||
)
|
||||
extra = values.get("extraArgs") or []
|
||||
extra_s = " ".join(f"{k}={v}" for k, v in extra.items()) if isinstance(extra, dict) \
|
||||
else " ".join(map(str, extra))
|
||||
check("enable-legacy-annotation-prefix" not in extra_s and "annotation-prefix" not in extra_s,
|
||||
"(8b) external-dns extraArgs must not set --annotation-prefix or "
|
||||
"--enable-legacy-annotation-prefix; the prefix is declared ONLY via values.annotationPrefix "
|
||||
"so this check can see it")
|
||||
|
||||
foreign: list[str] = []
|
||||
seen_keys = 0
|
||||
for sub in SCAN_DIRS:
|
||||
base = ROOT / sub
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in sorted(base.rglob("*.yaml")):
|
||||
rel = p.relative_to(ROOT).as_posix()
|
||||
for d in docs_of(p):
|
||||
for k in annotations_of(d):
|
||||
if isinstance(k, str) and EXTERNAL_DNS_KEY_RE.match(k):
|
||||
seen_keys += 1
|
||||
if not k.startswith(ANNOTATION_PREFIX):
|
||||
foreign.append(f"{rel}:{'/'.join(ident(d))} {k}")
|
||||
check(seen_keys > 0, "(8c) no external-dns annotation found anywhere — parse contract broken "
|
||||
"(the Gateway target alone guarantees at least one)")
|
||||
check(not foreign,
|
||||
f"(8c) external-dns annotations under a prefix the controller does NOT read "
|
||||
f"({ANNOTATION_PREFIX!r} is configured): {foreign}")
|
||||
render_src = read(CATALOG / "render.py")
|
||||
bad_render = sorted({m for m in re.findall(r"external-dns(?:\.[a-z0-9-]+)*\.kubernetes\.io/",
|
||||
render_src) if m != ANNOTATION_PREFIX})
|
||||
check(not bad_render,
|
||||
f"(8d) catalog/render.py emits external-dns keys under {bad_render}, but the controller "
|
||||
f"reads {ANNOTATION_PREFIX!r}")
|
||||
|
||||
# =============================================================================================
|
||||
if FAILURES:
|
||||
for f in FAILURES:
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: feat/l0-externaldns-annotation-prefix
|
||||
title: L0 guard — external-dns annotationPrefix must match every external-dns annotation in the tree
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
`conformance/tests/l0-static/internal-services-publish-no-public-dns/check.sh` gains assertion
|
||||
(8). The check is already whole-repo-scoped, so it runs on every gate.
|
||||
|
||||
- **(8a)** `platform/externaldns/helmrelease.yaml` must set `values.annotationPrefix` explicitly,
|
||||
and it must equal the check's `ANNOTATION_PREFIX` (`external-dns.alpha.kubernetes.io/`). The
|
||||
check does not trust the upstream default, because that default is what changed.
|
||||
- **(8b)** `extraArgs` must not carry `--annotation-prefix` or `--enable-legacy-annotation-prefix`,
|
||||
so the prefix has one declared place.
|
||||
- **(8c)** Every external-dns-looking annotation key under apps/, platform/ and runbooks/ must use
|
||||
that prefix. The match is any `external-dns(.x)*.kubernetes.io/` key.
|
||||
- **(8d)** catalog/render.py must emit no external-dns key under any other prefix.
|
||||
|
||||
`CONTROLLER_KEY` and `HOSTNAME_KEY` are now derived from `ANNOTATION_PREFIX`. The Service
|
||||
inventory matches any external-dns prefix variant, not only the alpha one.
|
||||
|
||||
Why: on 2026-11-07, external-dns v0.23.0 (whose default prefix is GA) ignored every alpha
|
||||
annotation. Assertions (1) to (3) stayed green, because they prove the annotations are present,
|
||||
not that the controller reads them. See `decisions/2026-11-07-public-dns-wan-a-record.md`.
|
||||
|
||||
This branch is stacked on `fix/public-dns-wan-a-record` (PR #140). Without that fix, (8a) fails,
|
||||
which is correct. Merge #140 first.
|
||||
|
||||
## What I verified
|
||||
|
||||
Run on the host (PyYAML):
|
||||
|
||||
- the check passes on this tree: `check OK: 62 assertions passed`.
|
||||
- it **fails (8a)** with origin/main's HelmRelease, which has no `annotationPrefix`.
|
||||
- it **fails (8c)** when `platform/gateway/gateway.yaml`'s `target` key is changed to the GA
|
||||
prefix.
|
||||
|
||||
Toolbox gate: see the PR description.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- Whether a future chart could read the prefix from somewhere other than `values.annotationPrefix`,
|
||||
for example a renamed chart value. (8a) would then pass while the flag silently vanished from the
|
||||
rendered args. Rendering the chart in the gate would close that gap. Not done: it needs network
|
||||
access or a vendored chart in L0.
|
||||
- `conformance/**` is a **sensitive path**. This PR needs the operator's approving review or
|
||||
`/approve-override`. The agent did not post one.
|
||||
- The header of the sibling chainsaw-test.yaml still says "external-dns v0.21.0". That is cosmetic
|
||||
and I left it alone.
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: fix/public-dns-wan-a-record
|
||||
title: external-dns v0.23 ignored every alpha-prefixed annotation — pin annotationPrefix back
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
`platform/externaldns/helmrelease.yaml`:
|
||||
|
||||
- adds `values.annotationPrefix: "external-dns.alpha.kubernetes.io/"`. The chart renders it as
|
||||
`--annotation-prefix=`.
|
||||
- tightens the chart range from `<1.24.0` to `>=1.23.0 <1.24.0`, because that flag needs app
|
||||
>= 0.22.
|
||||
- adds a comment block explaining why.
|
||||
|
||||
**Root cause.** Renovate PR #129 (merged 12:12:00Z) moved the chart from `>=1.21.0 <1.22.0` to
|
||||
`<1.24.0`. Helm deployed chart 1.23.0 at 12:13:08Z, which moved the app from v0.21.0 to v0.23.0.
|
||||
The upstream v0.22.0 release notes (#6424) say: "The default annotation prefix is now
|
||||
`external-dns.kubernetes.io/` with no fallback". Every external-dns annotation in this repo uses
|
||||
`external-dns.alpha.kubernetes.io/`. At 12:13:12Z v0.23.0 therefore:
|
||||
|
||||
- **ignored the Gateway's `target`.** Every public name flipped from `CNAME
|
||||
wan.home.keithjackson.dev` to `A 192.168.2.20`, an RFC1918 address served in public DNS.
|
||||
- **ignored `controller: none`.** The internal names jackett, pihole and semaphore were published.
|
||||
That breaks R-DNS-03.
|
||||
- **ignored the L4 Services' `hostname`.** The unifi and crafty-mc A records were deleted.
|
||||
|
||||
cloudflare-ddns (#123, v1.18.0) is not involved. It logs only `IPv4-enabled domains:
|
||||
wan.home.keithjackson.dev` and has made no changes.
|
||||
|
||||
**Why pin the prefix instead of migrating the annotations.** The pin is one value in one file, and
|
||||
that file is not on the sensitive-path list. It restores the exact v0.21 reading of `target`,
|
||||
`controller` and `hostname` together. A migration would touch `catalog/render.py`, every
|
||||
`apps/wave-*` file, the hand-written platform manifests, and the conformance goldens and checks.
|
||||
`conformance/**` needs an operator override. And a migration half-applied in either direction
|
||||
reproduces this incident.
|
||||
|
||||
I rejected `--enable-legacy-annotation-prefix`. Upstream calls it a migration aid that will be
|
||||
removed.
|
||||
|
||||
**Follow-up.** The L0 guard that the tree's annotation prefix matches this value is a separate PR
|
||||
(`feat/l0-externaldns-annotation-prefix`), because it lives in `conformance/**`. A full migration
|
||||
to the GA prefix remains open. It must change the annotations and this value in one commit.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **Live HelmRelease history.** `1.23.0 0.23.0 lastDeployed 2026-11-07T12:13:08Z deployed`, and
|
||||
before that `1.21.1 0.21.0 ... superseded`. The Deployment image is `external-dns:v0.23.0`. The
|
||||
startup config logs `AnnotationPrefix:external-dns.kubernetes.io/
|
||||
EnableLegacyAnnotationPrefix:false`.
|
||||
- **external-dns log, 12:13:12Z:**
|
||||
- `action=CREATE ... type=A` and `a-` TXT for gitea, crafty, the apex, headlamp, auth, jackett,
|
||||
nextcloud, keycloak, grafana, seerr, plex, pihole and semaphore.
|
||||
- `action=DELETE ... type=CNAME` and `cname-` TXT for all of those except jackett, pihole and
|
||||
semaphore.
|
||||
- `action=DELETE ... type=A` for unifi and crafty-mc.
|
||||
- **Before the upgrade.** Loki covers about 11-05 13:00Z to 12:13Z. The old pod logged no `Changing
|
||||
record` line in that window. Its only error was one Cloudflare 502 on 11-05.
|
||||
- **Gateway.** `gateway/shared` still carries `external-dns.alpha.kubernetes.io/target:
|
||||
wan.home.keithjackson.dev`.
|
||||
- **Public DNS.** At the authority (betty.ns.cloudflare.com), 1.1.1.1 and 8.8.8.8:
|
||||
- plex, gitea, auth, the apex, jackett, pihole and semaphore answer A `192.168.2.20`.
|
||||
- unifi and crafty-mc have no record.
|
||||
- `wan.` answers A `136.51.115.252`.
|
||||
- There is no AAAA for any of these names.
|
||||
- The `a-plex` and `a-jackett` TXT records read `external-dns/owner=prod`.
|
||||
- **Upstream v0.23.0 source.**
|
||||
- `controller/execute.go` calls `annotations.SetAnnotationPrefix(cfg.AnnotationPrefix)`.
|
||||
- `SetAnnotationPrefix` rebuilds `TargetKey`, `ControllerKey` and `HostnameKey`.
|
||||
- `source/gateway.go` still reads `TargetsFromTargetAnnotation(gw.Annotations)`.
|
||||
- Chart 1.23.0 `templates/deployment.yaml` renders `--annotation-prefix={{
|
||||
.Values.annotationPrefix }}`.
|
||||
- **Phone.** Pi-hole 192.168.2.23 answers correctly: A `192.168.2.20` with TTL 60, and AAAA
|
||||
NOERROR with no data. The router 192.168.2.1 answers `gitea.` with NOERROR and 0 answers, because
|
||||
it strips RFC1918 results. The workstation uses only `nameserver 192.168.2.23` and has
|
||||
`disable_ipv6 = 1`.
|
||||
- **Toolbox gate.** See the PR description.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **The fix live.** This depends on Flux applying it. Expected on the first reconcile, with
|
||||
`policy: sync` and owner `prod`, all of these records being prod-owned:
|
||||
- DELETE the A record and create CNAME `wan.` for the ten public names.
|
||||
- DELETE the A and TXT records for jackett, pihole and semaphore.
|
||||
- CREATE A for unifi (.21) and crafty-mc (.22).
|
||||
This is UNVERIFIED until someone reads the external-dns log and checks with `dig
|
||||
@betty.ns.cloudflare.com`. The A-to-CNAME type swap went through cleanly on 2026-10-03, and the
|
||||
reverse went through today, so I expect it to converge. If it does not, see the manual list in
|
||||
the PR.
|
||||
- **That the chart's schema accepts `annotationPrefix` under Flux.** I did not render the
|
||||
HelmRelease with helm here. The gate checks YAML and policy, not the chart render.
|
||||
- **The phone.** Not investigated, by decision. Likely cause: the phone bypasses Pi-hole on the
|
||||
client side. Candidates are IPv6 RA/RDNSS DNS from the router, Android Private DNS and iCloud
|
||||
Private Relay. Answers it gets that way are rebind-stripped, or point to the WAN IP, which needs
|
||||
hairpin NAT. Hairpin looked absent on 2026-10-03 (a self-signed cert on WAN:443). This is
|
||||
router- or client-side, not in-repo.
|
||||
- The AAAA records the operator saw on 8.8.8.8. I did not reproduce them.
|
||||
- The cilium (#104) and pihole (#109) bumps were not examined beyond Pi-hole answering correctly.
|
||||
@@ -11,7 +11,10 @@ spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: external-dns
|
||||
version: ">=1.21.0 <1.22.0" # k8s-1.36 catch-up (app v0.21; policy already set). Renovate maintains
|
||||
# chart 1.23.x = app v0.23.0. Floor >=1.23.0 because `annotationPrefix` below needs app >=0.22.
|
||||
# Renovate maintains — but READ THE APP RELEASE NOTES on every 0.x minor: 0.22 changed the
|
||||
# annotation prefix with no fallback and silently re-published every internal name (2026-11-07).
|
||||
version: ">=1.23.0 <1.24.0"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: external-dns
|
||||
@@ -25,6 +28,29 @@ spec:
|
||||
values:
|
||||
provider:
|
||||
name: cloudflare
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# ANNOTATION PREFIX — PINNED TO THE LEGACY `alpha` PREFIX (2026-11-07). Do not remove.
|
||||
#
|
||||
# external-dns v0.22.0 (#6424) changed the default annotation prefix from
|
||||
# `external-dns.alpha.kubernetes.io/` to `external-dns.kubernetes.io/` "with no fallback".
|
||||
# EVERY external-dns annotation in this repo uses the alpha prefix: the Gateway's `target`
|
||||
# (platform/gateway/gateway.yaml), the renderer's `controller: none` on `internal` routes
|
||||
# (catalog/render.py -> apps/wave-*), and the L4 Services' `hostname` (crafty-mc, unifi).
|
||||
# Renovate PR #129 moved the chart range to `<1.24.0` (app v0.21.0 -> v0.23.0) and on the first
|
||||
# reconcile (2026-11-07T12:13Z) v0.23.0, reading only the GA prefix, saw none of them:
|
||||
# * `target` ignored -> every public name flipped CNAME wan.<base> -> A 192.168.2.20 (RFC1918)
|
||||
# * `controller: none` ignored -> internal jackett, pihole, semaphore PUBLISHED (R-DNS-03 broken)
|
||||
# * `hostname` ignored -> unifi and crafty-mc A records DELETED
|
||||
# Logged as `action=CREATE ... type=A` / `action=DELETE ... type=CNAME` for every name at once.
|
||||
#
|
||||
# The chart's `annotationPrefix` renders `--annotation-prefix=`, which v0.23.0 applies to every
|
||||
# key it reads (source/annotations/annotations.go SetAnnotationPrefix: target, controller,
|
||||
# hostname, ttl, ...), restoring exactly the v0.21 behaviour. Chosen over
|
||||
# `--enable-legacy-annotation-prefix` (a dual-read migration aid upstream says will be removed)
|
||||
# and over migrating the annotations (renderer + goldens + conformance = sensitive paths, and a
|
||||
# flag-day across ~20 objects). Migrating to the GA prefix is the follow-up: change every
|
||||
# annotation AND this value in ONE commit, never one without the other.
|
||||
annotationPrefix: "external-dns.alpha.kubernetes.io/"
|
||||
# Watch Gateway API routes (Cilium Gateway) + Services. docs/09: records rendered from intent.
|
||||
#
|
||||
# BOTH SOURCES ARE UNFILTERED HERE, AND THAT IS DELIBERATE — see the WHICH RESOURCES block
|
||||
|
||||
Reference in new issue
Block a user