diff --git a/.gitignore b/.gitignore index aae8ddf..48dfe88 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ infra/tofu/.r2.env # Vendored ansible collections (installed by ansible-galaxy for node re-provision; not tracked or linted) infra/ansible/collections/ + +# standup.sh DNS snapshots — run artifacts, not authored state +.standup-dns/ diff --git a/bootstrap/standup.sh b/bootstrap/standup.sh index b38025d..da111a6 100755 --- a/bootstrap/standup.sh +++ b/bootstrap/standup.sh @@ -53,8 +53,12 @@ cd "${repo_root}" export KUBECONFIG="${KUBECONFIG_ARG}" [ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; } -# Where DNS snapshots go. Deliberately OUTSIDE the repo: they are run artifacts, not authored state. -SNAP_DIR="${STANDUP_SNAP_DIR:-${TMPDIR:-/tmp}/standup-dns}" +# Where DNS snapshots go. This MUST be a path that survives between invocations: each `dev.sh run` is +# a FRESH container, so a $TMPDIR default silently threw the baseline away and made `verify`'s +# before/after comparison — the check that detects damage to the OTHER cluster's records — impossible +# to run across steps. The repo is bind-mounted, so a repo-relative dir persists on the host. +# Gitignored: these are run artifacts, not authored state. +SNAP_DIR="${STANDUP_SNAP_DIR:-${repo_root}/.standup-dns}" mkdir -p "${SNAP_DIR}" say() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } @@ -124,7 +128,14 @@ dns_snapshot() { # [ "${page}" -ge "${tp:-1}" ] && break page=$((page+1)) done - grep -vE '^\s*$' "${out}" | sort -o "${out}" + # Strip CRs, and sort under a FIXED collation. Both matter, and both were learned the hard way: + # - a stray \r makes every line differ from a baseline captured without one; + # - locale-dependent sort ordering puts `*.keithjackson.dev` in a different position on the host + # than in the container, so a diff reports pairs of IDENTICAL lines as changed. That produced a + # "RECORDS OUTSIDE ... CHANGED" alarm on a zone where nothing had changed at all. + # A regression check that cries wolf is worse than none — it trains you to skip the one alarm that + # matters. LC_ALL=C makes the ordering reproducible everywhere. + tr -d '\r' < "${out}" | grep -vE '^\s*$' | LC_ALL=C sort -o "${out}" info "$(wc -l < "${out}") records -> ${out}" } @@ -395,10 +406,40 @@ step_verify() { say "gateway / LoadBalancer VIPs" kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true say "published DNS for this cluster's own domain" + local owner; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)" dns_snapshot "${SNAP_DIR}/dns-verify.txt" info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)" - warn "confirm every TXT registry entry for ${base} reads this cluster's owner id ($(cvar EXTERNALDNS_TXT_OWNER_ID)), - and that NO record outside ${base} was touched. Compare against ${SNAP_DIR}/dns-before.txt." + + # Every TXT registry entry this cluster publishes must carry OUR owner id. One reading another + # cluster's id means we have claimed its records and `policy: sync` will delete them next cycle. + local foreign + foreign="$(grep "${base}" "${SNAP_DIR}/dns-verify.txt" | grep 'heritage=external-dns' \ + | grep -v "external-dns/owner=${owner}" || true)" + if [ -n "${foreign}" ]; then + warn "TXT entries under ${base} NOT owned by '${owner}' — this cluster has claimed records it does not own:" + printf '%s\n' "${foreign}" | sed 's/^/ /' + else + info "every TXT registry entry under ${base} reads owner=${owner}: OK" + fi + + # THE REGRESSION CHECK. Additions under our own BASE_DOMAIN are expected. ANY change to a name + # outside it means we have touched the other cluster's zone — the 2026-09-04 incident. + if [ -f "${SNAP_DIR}/dns-before.txt" ]; then + say "regression check against the pre-standup baseline" + local outside + outside="$(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" \ + | grep -E '^[<>]' | grep -v "${base}" || true)" + if [ -n "${outside}" ]; then + warn "RECORDS OUTSIDE ${base} CHANGED — investigate before doing anything else:" + printf '%s\n' "${outside}" | sed 's/^/ /' + else + info "no record outside ${base} was added, changed or removed: OK" + info "added under ${base}: $(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" | grep -c '^>' || true)" + fi + else + warn "no ${SNAP_DIR}/dns-before.txt baseline — cannot prove the other cluster's records are untouched. + Take one BEFORE the next standup so this check can run." + fi } # ============================================================================ dispatch diff --git a/infra/PROD-STANDUP.md b/infra/PROD-STANDUP.md index 209fb84..3f3e066 100644 --- a/infra/PROD-STANDUP.md +++ b/infra/PROD-STANDUP.md @@ -220,6 +220,45 @@ timeout 3600 infra/devcontainer/dev.sh run bootstrap/standup.sh \ --- +## VERIFIED WORKING — 2026-09-05, standup complete + +| | | +|---|---| +| Kustomizations | **12/13 Ready** (`velero` deliberately suspended — see §Blockers) | +| Pods | **113 total, 0 not Running/Completed** | +| Wildcard cert | `wildcard-home` **Ready**, Let's Encrypt **prod**, `*.prod.home.keithjackson.dev` | +| Gateway VIP | `192.168.2.20` · crafty L4 `192.168.2.22` · unifi L4 `192.168.2.21` | +| Published DNS | **36 records** under `prod.home.keithjackson.dev`, every TXT registry entry `owner=prod` | +| Dev's records | **57, byte-identical to the pre-standup baseline — nothing added, changed or removed** | + +End-to-end HTTPS, certificate validated (no `-k`), pinned to prod's gateway: + +``` +prod.home.keithjackson.dev 200 keycloak.… 302 (login redirect) +gitea.prod.home.keithjackson.dev 200 grafana.… 302 (SSO redirect) +``` + +### ACTION REQUIRED — the pfSense override shadows prod from the LAN + +`getent hosts` on the workstation returns **192.168.2.50** (dev) for **every** `*.home.keithjackson.dev` +name — and the new `prod.home.…` subtree falls inside that wildcard. Cloudflare correctly returns +`192.168.2.20`. So **prod is reachable from anywhere using public DNS, but from this workstation and +anything else behind pfSense, prod hostnames resolve to dev.** + +Narrow the override so it does not cover `prod.home.keithjackson.dev`, or add a matching override +pointing that subtree at `192.168.2.20`. Until then, test prod with an explicit resolve: + +```bash +curl -sS --resolve gitea.prod.home.keithjackson.dev:443:192.168.2.20 \ + https://gitea.prod.home.keithjackson.dev/ +``` + +**This is also the trap that produced a false healthy reading during the 2026-09-04 incident.** It is +subtle because the *apex* appears to work: dev's `*.home.keithjackson.dev` wildcard matches +`prod.home.keithjackson.dev` (one label) so TLS succeeds, while `keycloak.prod.home.keithjackson.dev` +(two labels) fails with a certificate-name error. A partial success is what makes it convincing. +**Never verify DNS or TLS through a local resolver here.** + ## Blockers ### 1. `S3_BACKUP_ENDPOINT` is an intentional placeholder — operator decision required