Compare commits
245
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1adc3963c | ||
|
|
4ca0ebfac6 | ||
|
|
1ff527f546 | ||
|
|
d8b25a67fe | ||
|
|
1763e27859 | ||
|
|
4c82c64b2f | ||
|
|
d06dff9a19 | ||
|
|
b1a6fe6503 | ||
|
|
b265d08278 | ||
|
|
2b9dcef6c7 | ||
|
|
d4bea8a78c | ||
|
|
b85d1e3ed7 | ||
|
|
50ec2297ee | ||
|
|
ec6b39bb91 | ||
|
|
c3c72c47bb | ||
|
|
4c3a284cad | ||
|
|
4a89aacfbc | ||
|
|
50c8719b12 | ||
|
|
8f60e6103a | ||
|
|
da0515c8c8 | ||
|
|
3a2ec5a12c | ||
|
|
f13d336630 | ||
|
|
8fe8d5df8a | ||
|
|
020e736541 | ||
|
|
92944486d1 | ||
|
|
2e777e13ca | ||
|
|
063c8acf77 | ||
|
|
16222606ae | ||
|
|
d9ab57f2d1 | ||
|
|
9724445a08 | ||
|
|
32b7d5bac1 | ||
|
|
83e0ac194f | ||
|
|
09f219fe7f | ||
|
|
aabc8c60b6 | ||
|
|
eb62c13c6b | ||
|
|
4226292dd6 | ||
|
|
213429a41f | ||
|
|
0120bc10d4 | ||
|
|
d1963bd000 | ||
|
|
0b0b5abc8e | ||
|
|
6e2e353bcc | ||
|
|
045d81c7a2 | ||
|
|
7be693bf37 | ||
|
|
9e9a3ea8cc | ||
|
|
89f48e4a5f | ||
|
|
64c5617ddd | ||
|
|
35ba875380 | ||
|
|
27d603f726 | ||
|
|
a775082ae1 | ||
|
|
1a33814522 | ||
|
|
4a3f6dee59 | ||
|
|
514ed5d5a4 | ||
|
|
9537ca2ddb | ||
|
|
b548224db8 | ||
|
|
759a0e51c8 | ||
|
|
ea30ba876c | ||
|
|
3d7ecbcfe9 | ||
|
|
e8d26669a4 | ||
|
|
048254bfaf | ||
|
|
dd94101647 | ||
|
|
60e3a24652 | ||
|
|
8a9baa92bc | ||
|
|
2d32b9bd7d | ||
|
|
9f82b2accf | ||
|
|
990b309cbb | ||
|
|
cedcb01975 | ||
|
|
2b279d4f21 | ||
|
|
12c28adf4d | ||
|
|
b0230ba736 | ||
|
|
9c51f60a35 | ||
|
|
267446c68f | ||
|
|
d1994b16e8 | ||
|
|
d7317606e7 | ||
|
|
970d65d193 | ||
|
|
6719082e5c | ||
|
|
428b77f08c | ||
|
|
2ee7d5036a | ||
|
|
5736f42e22 | ||
|
|
773254e3a1 | ||
|
|
264cd7a46d | ||
|
|
979cc4b21d | ||
|
|
be6742c22b | ||
|
|
c8b915d8f2 | ||
|
|
6cd3bd29a9 | ||
|
|
2040d35222 | ||
|
|
47fc27ff08 | ||
|
|
5e8b080298 | ||
|
|
24a0e1e4c5 | ||
|
|
3aec6886d8 | ||
|
|
1b2ec82a9d | ||
|
|
3724d3068b | ||
|
|
c6ad6b559c | ||
|
|
6efc49d415 | ||
|
|
893c9dee4f | ||
|
|
a6dc2d1adb | ||
|
|
75a472cf67 | ||
|
|
24b02f37df | ||
|
|
be3ce4645b | ||
|
|
619c2943da | ||
|
|
b7a77a7d1e | ||
|
|
1a94b0535c | ||
|
|
0e7d0204b3 | ||
|
|
b77572f843 | ||
|
|
76b5d038fb | ||
|
|
306f6f4fb0 | ||
|
|
3ceaad15e0 | ||
|
|
ae772e050d | ||
|
|
517dfc9030 | ||
|
|
0605ecf593 | ||
|
|
16de787557 | ||
|
|
b7cb70955c | ||
|
|
9bcea04b1b | ||
|
|
e8e9de8611 | ||
|
|
2fdb95393c | ||
|
|
5b484f6b78 | ||
|
|
640edb758d | ||
|
|
e6cd064f5c | ||
|
|
dd9d8da86a | ||
|
|
c3ce4db767 | ||
|
|
38076f1cff | ||
|
|
552330398b | ||
|
|
e49aebc803 | ||
|
|
9c0292001e | ||
|
|
db9416d3b5 | ||
|
|
cb4a1bf2de | ||
|
|
028f5229f0 | ||
|
|
44e2cd3c64 | ||
|
|
b30b7e6f67 | ||
|
|
5a3a5f8df2 | ||
|
|
f698bc82a0 | ||
|
|
6bc10fb60d | ||
|
|
d8c330fb8c | ||
|
|
a1d89f4a5f | ||
|
|
19097e4d4f | ||
|
|
466f63ec3c | ||
|
|
664ee82b9a | ||
|
|
95e08da2f0 | ||
|
|
0d3e51acc4 | ||
|
|
11267409d7 | ||
|
|
6a192ede2a | ||
|
|
06d4a9a1fc | ||
|
|
9db20e8ca0 | ||
|
|
ddd04d460e | ||
|
|
f0760d1024 | ||
|
|
553d027f7c | ||
|
|
183bca6897 | ||
|
|
ce849032d5 | ||
|
|
b766130b85 | ||
|
|
862773f31b | ||
|
|
db4ba79a1c | ||
|
|
0b6282aba4 | ||
|
|
c6cfaf62b6 | ||
|
|
4ae648f6d3 | ||
|
|
a4d5206294 | ||
|
|
47eb6afa02 | ||
|
|
74ba90c0b3 | ||
|
|
fb45d7441b | ||
|
|
8ed4e30201 | ||
|
|
24db269ea8 | ||
|
|
df96be98bf | ||
|
|
b4ff2879be | ||
|
|
d9571fc6d8 | ||
|
|
593d0fad15 | ||
|
|
0c6ee4921d | ||
|
|
d550a501e2 | ||
|
|
365e43cdea | ||
|
|
7f0cbc7ba2 | ||
|
|
a7eadd4054 | ||
|
|
dfc432c6ea | ||
|
|
262c6e4a97 | ||
|
|
523cd9118f | ||
|
|
d2c4bd5194 | ||
|
|
0dd57fbaf0 | ||
|
|
04eb38584a | ||
|
|
f65c12b395 | ||
|
|
8f35234fcc | ||
|
|
e539109830 | ||
|
|
27da5555a4 | ||
|
|
da1b36eb64 | ||
|
|
6bb8af1d70 | ||
|
|
7c4eb439b2 | ||
|
|
17eed6e8ae | ||
|
|
f5931944fa | ||
|
|
aacf361bb8 | ||
|
|
1301af9a4c | ||
|
|
42ce001d71 | ||
|
|
84bb0f0acd | ||
|
|
0d4bd7297d | ||
|
|
713b0c3dba | ||
|
|
fda5c31a94 | ||
|
|
c340bbfc4b | ||
|
|
a9a529750d | ||
|
|
793c3940a8 | ||
|
|
ef8ef5ac29 | ||
|
|
ee196c7bcd | ||
|
|
83d3568150 | ||
|
|
24bea918ab | ||
|
|
5642881c4d | ||
|
|
4d489f2059 | ||
|
|
a2440a3125 | ||
|
|
16894b7b73 | ||
|
|
1a0b20857a | ||
|
|
c150327a30 | ||
|
|
2f2e98aa19 | ||
|
|
9a41a8e1e5 | ||
|
|
0132c680b3 | ||
|
|
439b35a3f0 | ||
|
|
e89df78210 | ||
|
|
0747849837 | ||
|
|
ad37acdf53 | ||
|
|
2acfed460a | ||
|
|
e1bebb857e | ||
|
|
23db97b958 | ||
|
|
5928d8153c | ||
|
|
146dd9a906 | ||
|
|
d1c07fe021 | ||
|
|
5c43fbac65 | ||
|
|
cf7d5e4316 | ||
|
|
be23cf3e26 | ||
|
|
4791653f2b | ||
|
|
523f482957 | ||
|
|
8059297036 | ||
|
|
90a79d29d3 | ||
|
|
523a463fd7 | ||
|
|
e35612dc7e | ||
|
|
a209a618fd | ||
|
|
6253d546f8 | ||
|
|
95597bd688 | ||
|
|
06ca9eeb88 | ||
|
|
18f741d744 | ||
|
|
867a82d8ad | ||
|
|
e4a9b02b00 | ||
|
|
0569d47d49 | ||
|
|
3ee663dbcc | ||
|
|
ad3f56eee8 | ||
|
|
005e80d38a | ||
|
|
8b5e950baa | ||
|
|
2cb891c05d | ||
|
|
54418363aa | ||
|
|
f92707f38c | ||
|
|
c5b5b8672a | ||
|
|
22c94effff | ||
|
|
d0db63911a | ||
|
|
81f627167d | ||
|
|
4386d1453b |
No files matched your search
@@ -115,14 +115,63 @@ CI. That is a real coverage gap, not a phase.
|
||||
| variable | `REQUIRE_CLUSTER_TESTS` | `'true'` (Phase 1+) makes unverified cluster-needing touched reqs a hard block |
|
||||
| secret | `CONFORMANCE_TRIGGER_KUBECONFIG` | narrow creds to create/await the in-cluster Chainsaw Job (not cluster-admin) |
|
||||
|
||||
## How images are built
|
||||
|
||||
This repo's three images — `toolbox` (`infra/devcontainer/`), `agent-runner` (`infra/agent/`,
|
||||
built `FROM` the toolbox) and `conformance-runner` (`conformance/incluster/`) — are built,
|
||||
verified and published by [`images.yaml`](images.yaml), never from a workstation
|
||||
(operator rule, 2026-10-29). The logic is in [`scripts/images.sh`](scripts/images.sh).
|
||||
|
||||
**The builder is rootless BuildKit** ([`platform/buildkit/`](../../platform/buildkit/README.md)),
|
||||
reached over mTLS with a pinned `buildx` — no job here touches a Docker daemon, privileged or
|
||||
otherwise (operator decision on PR #81). The client certificate reaches jobs as the Actions
|
||||
secrets `BUILDKIT_CA`, `BUILDKIT_CLIENT_CERT`, `BUILDKIT_CLIENT_KEY`.
|
||||
|
||||
**Verification is a build target.** Each Containerfile ends in a `verify` stage that runs every
|
||||
in-image assertion at build time as the non-root uid with `RUN --network=none` (tool presence and
|
||||
pinned versions, `PM_SELFTEST=1 pm-dispatch`, the pm-worker probe, the `conformance-clone`
|
||||
refusals, no writable path outside the workspace). The pipeline builds `<image>-verify` before it
|
||||
builds, let alone pushes, the image. Build args come from each image's own
|
||||
`build-image.sh --print-build-args`, which is also the local podman path — so the recipe and the
|
||||
proof exist once.
|
||||
|
||||
| Trigger | What happens |
|
||||
|---|---|
|
||||
| pull request touching an image's inputs | build + verify only. Nothing is pushed and **no registry credential is referenced** |
|
||||
| push to `main` touching an image's inputs | build + verify + publish to `gitea.home.keithjackson.dev/home-network/<image>` as `sha-<commit>` (immutable) and `main` (moving); the **registry-returned** digest is in the job summary |
|
||||
| `workflow_dispatch` | the same as a push when run on `main` (all images by default); verify-only elsewhere |
|
||||
|
||||
Only images whose inputs changed are built. The publish job **never writes to git**: Renovate
|
||||
follows each image's `main` tag through a `# renovate-digest:` marker next to every digest
|
||||
reference (`platform/pm-agent/`, `infra/agent/versions.env`, `conformance/versions.env`) and
|
||||
opens the bump PR, which goes through the gate like anything else. A PR opened with the Actions
|
||||
token would not trigger the required gate, which is why it is Renovate's job and not this one's.
|
||||
|
||||
Credential: the repository secret **`REGISTRY_PUSH_TOKEN`** (scope `write:package`, owned by an
|
||||
account with package write on the org). The automatic Actions token cannot push in Gitea 1.24.3
|
||||
— the reasoning, with source citations, is in the header of `images.yaml`. Retention:
|
||||
[`infra/gitea/registry-retention.sh`](../../infra/gitea/registry-retention.sh), an operator
|
||||
action, keeps every digest git pins plus the newest tags.
|
||||
|
||||
Cost: the job shares the single act_runner slot with the gate, but the build itself runs in the
|
||||
`buildkitd` pod, not in DinD, so the runner's disk and memory are untouched. Until
|
||||
`platform/buildkit` is deployed and its client certificate copied
|
||||
(`infra/gitea/buildkit-client-secret.sh`), the job fails loudly at its builder preflight.
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
.gitea/workflows/
|
||||
merge-gate.yaml the pipeline (lint → l0 → l1 → conformance → decision)
|
||||
images.yaml build/verify/publish this repo's container images (see above)
|
||||
cve-triage.yaml scheduled CVE triage bot (R-VULN-05/07, ADR-0027; runs before Renovate)
|
||||
scripts/
|
||||
install-tools.sh installs the pinned tools per stage (from ci/versions.env)
|
||||
install-tools.sh installs the pinned tools per stage (from ci/versions.env) — used by the
|
||||
dormant merge-gate.yaml blueprint only; gate.yml runs in the toolbox image
|
||||
decide.sh the R-TEST-03 merge decision over job results
|
||||
images.sh the image pipeline's driver (what to build, base, push, digest)
|
||||
cve_triage.py the CVE triage bot: reads the scanner report, maintains the counts-only
|
||||
CVE Dashboard issue, ticks Renovate approval rows (cve-triage.yaml)
|
||||
README.md this file
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
# CVE triage bot (R-VULN-05, R-VULN-07; docs/17 "The triage bot"; ADR-0027). Same shape as
|
||||
# renovate.yaml: scheduled, loud on a missing credential, never concurrent with itself, and it
|
||||
# records what it did in the job summary. The logic and its reasoning live in
|
||||
# scripts/cve_triage.py; this file only supplies the image, the code, the credential and the
|
||||
# schedule.
|
||||
#
|
||||
# WHY IT CANNOT SUCCEED SILENTLY. A skipped Gitea job reports `success` (CLAUDE.md §2), so the
|
||||
# token preflight exits 1 rather than skipping, and an unreachable scanner fails the script
|
||||
# rather than skipping. The heartbeat POST is the script's last action, so VulnTriageBotSilent
|
||||
# fires for every run that did not get all the way through, and when the schedule stops firing.
|
||||
#
|
||||
# THE ONE SECRET: RENOVATE_TOKEN, deliberately reused (ADR-0027). It acts as renovate-bot, which
|
||||
# is also what authored the dashboard this edits. It needs read:repository and write:issue.
|
||||
name: cve-triage
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 07:15 UTC, one hour before Renovate's 08:15 (renovate.yaml), so a tick is read the same
|
||||
# morning, and clear of its run so the bot's edit cannot race Renovate's rewrite of the body.
|
||||
- cron: "15 7 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run: read everything, write nothing, print counts."
|
||||
required: false
|
||||
default: "true"
|
||||
|
||||
concurrency:
|
||||
group: cve-triage
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
triage:
|
||||
runs-on: ubuntu-latest
|
||||
# THE TOOLBOX, the image gate.yml already runs, pinned by digest. A job here has no Docker
|
||||
# daemon, so a job container is the only way to choose an image. The toolbox provides
|
||||
# python3, git, curl and jq, pulls anonymously like the gate's, and is normally already in
|
||||
# the DinD store. It has no Node, so actions/checkout cannot run; a public python image has
|
||||
# neither Node nor git and would add a second image to keep pinned. The cost is a cold pull
|
||||
# after a runner restart, which the gate pays anyway.
|
||||
container:
|
||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
||||
image: gitea.home.keithjackson.dev/home-network/toolbox@sha256:83a3501efe34fa0b039e3f0d360738686a0f59109866c49779768f5d5ed0ef81
|
||||
# Without a bound, a wedged run would hold the concurrency group and block every later run.
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Preflight - the token must exist, and its absence must be LOUD
|
||||
shell: bash
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${RENOVATE_TOKEN:-}" ]; then
|
||||
echo "::error::RENOVATE_TOKEN is not set on this repository." >&2
|
||||
echo "A skipped job reports success, so this fails on purpose: without the token no dashboard is maintained and no approval is requested. Add the Actions secret RENOVATE_TOKEN (scopes read:repository, write:issue)." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "RENOVATE_TOKEN is present (value not printed)."
|
||||
|
||||
- name: Preflight - Gitea is reachable and the token can do this job
|
||||
shell: bash
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
# GETs only. The token reaches curl on stdin, never argv. A 401 is the token's value, a 403
|
||||
# its scope; the script's own errors would only show a bare status later.
|
||||
run: |
|
||||
set -uo pipefail
|
||||
GITEA=https://gitea.home.keithjackson.dev
|
||||
REPO=Home-Network/home-network-k3s-impl
|
||||
fail=0
|
||||
body=$(mktemp)
|
||||
fetch() {
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$RENOVATE_TOKEN" \
|
||||
| curl -sS --max-time 30 --config - -o "$body" -w '%{http_code}' "$GITEA$1" || true
|
||||
}
|
||||
probe() { # <what it proves> <path>
|
||||
local code
|
||||
code=$(fetch "$2")
|
||||
if [ "$code" = 200 ]; then
|
||||
echo "ok $1"
|
||||
else
|
||||
fail=1
|
||||
echo "::error::$1: GET ${2%%\?*} -> $code"
|
||||
fi
|
||||
}
|
||||
probe "read:repository, Gitea reachable" "/api/v1/repos/$REPO"
|
||||
if [ "$(jq -r '.permissions.push // false' "$body" 2>/dev/null)" != "true" ]; then
|
||||
fail=1
|
||||
echo "::error::renovate-bot has no write access to $REPO, so it cannot create labels or edit issues."
|
||||
fi
|
||||
probe "read:issue" "/api/v1/repos/$REPO/labels?limit=1"
|
||||
probe "Renovate's dashboard readable" "/api/v1/repos/$REPO/issues/91"
|
||||
# A GET cannot prove a write scope; Gitea's scope refusal lists the token's scopes
|
||||
# (names only). Same technique as renovate.yaml, advisory if the message format changes.
|
||||
fetch "/api/v1/admin/orgs?limit=1" > /dev/null
|
||||
scopes=$(jq -r '.message // empty' "$body" 2>/dev/null | sed -n 's/.*token scope=//p')
|
||||
if [ -n "$scopes" ]; then
|
||||
case ",$scopes," in
|
||||
*",write:issue,"*|*",all,"*) echo "ok write:issue" ;;
|
||||
*) echo "::error::scope write:issue is missing (token scope=$scopes)"; fail=1 ;;
|
||||
esac
|
||||
else
|
||||
echo "::warning::could not read the token's scope list; write:issue is UNVERIFIED"
|
||||
fi
|
||||
rm -f "$body"
|
||||
[ "$fail" -eq 0 ]
|
||||
|
||||
# Plain git for the same reason as gate.yml: the toolbox has no Node. One commit is enough,
|
||||
# and the job token goes to the single fetch only, never into .git/config.
|
||||
- name: Checkout the script (git, one commit)
|
||||
shell: bash
|
||||
env:
|
||||
GIT_TOKEN: ${{ github.token }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 0022
|
||||
git config --global --add safe.directory "$PWD"
|
||||
git init -q .
|
||||
git remote add origin "${SERVER_URL}/${REPOSITORY}"
|
||||
auth="$(printf 'x-access-token:%s' "$GIT_TOKEN" | base64 -w0)"
|
||||
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \
|
||||
GIT_CONFIG_VALUE_0="AUTHORIZATION: basic ${auth}" \
|
||||
git -c protocol.version=2 fetch --quiet --depth=1 --no-tags origin \
|
||||
"+${SHA}:refs/remotes/checkout/head"
|
||||
unset auth GIT_TOKEN
|
||||
git checkout -q --force --detach "$SHA"
|
||||
[ "$(git rev-parse HEAD)" = "$SHA" ]
|
||||
|
||||
- name: Triage
|
||||
shell: bash
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
GITEA_URL: https://gitea.home.keithjackson.dev
|
||||
GITEA_REPOSITORY: Home-Network/home-network-k3s-impl
|
||||
RENOVATE_DASHBOARD_ISSUE: "91"
|
||||
VULN_API_URL: http://vuln-scanner.vuln-scanner.svc.cluster.local:8081
|
||||
# The Service's pinned spec.clusterIP (platform/vuln-scanner/service.yaml), used only
|
||||
# when the name above fails to resolve. Keep the two in step.
|
||||
VULN_API_FALLBACK_URL: http://10.43.210.10:8081
|
||||
VULN_PAGE_URL: https://cve.home.keithjackson.dev
|
||||
# Anything other than an explicit false on a dispatch is a dry run; a schedule is live.
|
||||
DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run != 'false' && 'true' || 'false' }}
|
||||
run: python3 .gitea/workflows/scripts/cve_triage.py
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CVE triage"
|
||||
echo
|
||||
echo "- trigger: \`${{ github.event_name }}\`"
|
||||
echo "- dry run requested: \`${{ github.event_name == 'workflow_dispatch' && inputs.dry_run != 'false' && 'true' || 'false' }}\`"
|
||||
echo "- outcome: \`${{ job.status }}\`"
|
||||
echo
|
||||
echo "Counts only: which CVE lands where is on the CVE page, never here."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
+100
-11
@@ -3,8 +3,10 @@
|
||||
# This is the LIVE gate wired to the in-cluster Gitea Actions runner (ns gitea-actions,
|
||||
# Docker executor + DinD). It invokes the AGGREGATE merge gate — `./ci/gate.sh`, which is
|
||||
# exactly what `make -C ci gate` runs, so CI and the devcontainer gate the same way
|
||||
# (R-TEST-02): the runner just installs the pinned tools from ci/versions.env first (and see
|
||||
# below for why the script, not `make`, is invoked here). It runs every stage in order —
|
||||
# (R-TEST-02) — and since 2026-11-08 it runs INSIDE the same image: the job `container:` is the
|
||||
# CI-built devcontainer toolbox (see "THE JOB IMAGE" below), not a generic runner image that
|
||||
# downloads the pinned tools on every run. (See further below for why the script, not `make`,
|
||||
# is invoked here.) It runs every stage in order —
|
||||
# spec-immutability, lint, Kyverno L0, render==golden, L1 offline render — and then applies the
|
||||
# R-TEST-03 decision, printing
|
||||
# "MERGE GATE: PASS" (0) / "BLOCKED" (1) / "UNDETERMINED" (3) / "NOT APPLICABLE" (0). It is the
|
||||
@@ -17,7 +19,8 @@
|
||||
# kinds of non-run must be told apart HERE, in the caller, because only the caller knows which
|
||||
# environment it is:
|
||||
# * a pinned tool missing or a seam unwired ON THIS RUNNER is a DEFECT — the toolchain is
|
||||
# installed for us two steps up. GATE_STRICT_SKIPS=true DECLARES that (no env sniffing, no
|
||||
# baked into the job image and checked against ci/versions.env two steps up.
|
||||
# GATE_STRICT_SKIPS=true DECLARES that (no env sniffing, no
|
||||
# "does this look like CI?" heuristic) and makes those skips exit 1: untolerated, job fails.
|
||||
# * spec-immutability CANNOT be verified here at all: the spec repo is a separate repo and is
|
||||
# not checked out into this runner (SPEC_REPO_SLUG unwired; blueprint in merge-gate.yaml). It
|
||||
@@ -41,6 +44,32 @@
|
||||
# same script the laptop runs; the extra pre-step is a read-only git-diff check and is
|
||||
# purely additive. The full pipeline blueprint (spec-checkout, per-layer jobs, decision job)
|
||||
# is retained dormant in merge-gate.yaml.
|
||||
#
|
||||
# THE JOB IMAGE (2026-11-08, feat/gate-toolbox-container). The gate job runs in
|
||||
# gitea.home.keithjackson.dev/home-network/toolbox, pinned BY DIGEST — the image
|
||||
# .gitea/workflows/images.yaml builds from infra/devcontainer/Containerfile, verifies, and
|
||||
# publishes. It is the SAME image `infra/devcontainer/dev.sh` runs, so a local gate and the CI
|
||||
# gate now execute the identical toolchain (R-TEST-02), not two installs that agree by intent.
|
||||
# Before this, the job ran in ghcr.io/catthehacker/ubuntu:act-22.04 and spent ~11s of every run
|
||||
# re-downloading yamllint, check-jsonschema, kyverno, promtool and chainsaw
|
||||
# (.gitea/workflows/scripts/install-tools.sh l0 + l1; measured on runs 494/495).
|
||||
# * PINNED BY DIGEST, MOVED BY RENOVATE. The `# renovate-digest:` marker above the image line
|
||||
# is the repo-wide convention (renovate.json5, "This repo's own CI-published images"):
|
||||
# Renovate follows the `main` tag and opens a digest-bump PR, whose OWN gate run then
|
||||
# executes on the new image — the bump tests itself. infra/gitea/registry-retention.sh reads
|
||||
# the same marker, so the pinned digest is never pruned. Never hand-edit it to a tag.
|
||||
# * PULL AUTH: none needed. The Home-Network org is public and the registry issues an
|
||||
# anonymous pull token (measured 2026-11-08: anonymous /v2/token -> manifest 200), so the
|
||||
# runner's DinD pulls it like any public image. If the org is ever made private, add
|
||||
# `credentials:` here with a read:package token secret — the automatic Actions token has no
|
||||
# package scope at /v2/token (see images.yaml's header).
|
||||
# * COLD PULL: ~467 MB compressed (1.54 GB unpacked). The DinD store is an emptyDir, so the
|
||||
# first job after a runner-pod restart or a digest bump pays one pull; every other run finds
|
||||
# it cached (forcePull=false), exactly as it already did for the catthehacker image.
|
||||
# * NO NODE in the toolbox, so `actions/checkout` (a Node action) cannot run here. The checkout
|
||||
# step is plain git instead — see its comment for how it keeps fetch-depth-0 semantics.
|
||||
# * install-tools.sh is NOT deleted: merge-gate.yaml (the dormant blueprint) still calls it,
|
||||
# and conformance/tests/l0-static/alert-rules-unit-tested asserts its promtool wiring.
|
||||
---
|
||||
name: gate
|
||||
|
||||
@@ -78,16 +107,76 @@ jobs:
|
||||
# Single required status check: the static merge gate.
|
||||
gate:
|
||||
runs-on: ubuntu-latest
|
||||
# The CI-built toolbox — the same image dev.sh runs. See "THE JOB IMAGE" in the header.
|
||||
container:
|
||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
||||
image: gitea.home.keithjackson.dev/home-network/toolbox@sha256:83a3501efe34fa0b039e3f0d360738686a0f59109866c49779768f5d5ed0ef81
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # full history so ci/touched-requirements.sh can diff vs main
|
||||
|
||||
- name: Install pinned CI toolchain (ci/versions.env)
|
||||
# Plain git, because actions/checkout is a Node action and the toolbox has no Node. It
|
||||
# reproduces what actions/checkout@v7 with `fetch-depth: 0` did here (read from its logged
|
||||
# commands on runs 490/495): full history of every branch + tags, so
|
||||
# ci/touched-requirements.sh can diff against origin/<base>; the exact event commit
|
||||
# checked out (detached for a PR, as `-B <branch>` for a push). One difference, on
|
||||
# purpose: the job token is passed to the single fetch through GIT_CONFIG_* environment
|
||||
# variables and is NOT persisted into .git/config — nothing after this step needs
|
||||
# authenticated git (ci/ and conformance/ make no network git calls), so there is no
|
||||
# reason to leave a credential in the workspace for the gate's scripts to read.
|
||||
- name: Checkout (git, full history)
|
||||
env:
|
||||
GIT_TOKEN: ${{ github.token }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
REF: ${{ github.ref }}
|
||||
run: |
|
||||
./.gitea/workflows/scripts/install-tools.sh l0
|
||||
./.gitea/workflows/scripts/install-tools.sh l1
|
||||
set -euo pipefail
|
||||
# act runs steps with umask 0000, so without this every checked-out directory is
|
||||
# world-writable — and Ansible then IGNORES infra/ansible/ansible.cfg as a source
|
||||
# ("Ansible is being run in a world writable directory"), which made the L0 check
|
||||
# prod-baremetal-not-dev-surface fail on this PR's first CI run (run 500). The
|
||||
# toolbox has Ansible, so that check now drives it for real. 0022 = what a
|
||||
# workstation clone gets (reproduced locally: 0000 fails the check, 0022 passes).
|
||||
umask 0022
|
||||
git config --global --add safe.directory "$PWD"
|
||||
git init -q .
|
||||
git remote add origin "${SERVER_URL}/${REPOSITORY}"
|
||||
auth="$(printf 'x-access-token:%s' "$GIT_TOKEN" | base64 -w0)"
|
||||
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \
|
||||
GIT_CONFIG_VALUE_0="AUTHORIZATION: basic ${auth}" \
|
||||
git -c protocol.version=2 fetch --quiet --no-tags --prune --no-recurse-submodules origin \
|
||||
'+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' \
|
||||
"+${SHA}:refs/remotes/checkout/head"
|
||||
unset auth GIT_TOKEN
|
||||
case "$REF" in
|
||||
refs/heads/*) git checkout -q --force -B "${REF#refs/heads/}" "$SHA" ;;
|
||||
*) git checkout -q --force --detach "$SHA" ;;
|
||||
esac
|
||||
[ "$(git rev-parse HEAD)" = "$SHA" ] || { echo "::error::HEAD is not ${SHA}" >&2; exit 1; }
|
||||
echo "checked out $(git log -1 --format='%H %s') (ref ${REF}); $(git rev-list --count HEAD) commits of history"
|
||||
|
||||
# The image is the toolchain, so prove it is the PINNED one before trusting it: every
|
||||
# *_VERSION in ci/versions.env must equal what the image recorded at build time
|
||||
# (/etc/dev-toolchain-versions, written by infra/devcontainer/Containerfile; its `verify`
|
||||
# stage already asserted the version-sensitive binaries report those pins). This replaces
|
||||
# the per-run install: a drifted toolbox fails HERE, naming the tool, instead of gating with
|
||||
# a different engine. ci/versions.env and infra/devcontainer/versions.env must move together.
|
||||
- name: Toolchain is the one ci/versions.env pins
|
||||
run: |
|
||||
set -euo pipefail
|
||||
. ci/versions.env
|
||||
drift=0
|
||||
for var in $(grep -oE '^[A-Z_]+_VERSION' ci/versions.env); do
|
||||
tool="$(printf '%s' "${var%_VERSION}" | tr 'A-Z_' 'a-z-')"
|
||||
want="${!var}"
|
||||
got="$(sed -n "s/^${tool}=//p" /etc/dev-toolchain-versions)"
|
||||
if [ "$got" = "$want" ]; then
|
||||
printf 'ok %-18s %s\n' "$tool" "$want"
|
||||
else
|
||||
printf '::error::DRIFT %s: ci/versions.env pins %s, the job image has %s\n' "$tool" "$want" "${got:-nothing}"
|
||||
drift=1
|
||||
fi
|
||||
done
|
||||
[ "$drift" -eq 0 ]
|
||||
|
||||
# R-TEST-06: the spec is read-only to agents. An EXPLICIT early surfacing of the
|
||||
# spec-immutability check, so its result is a distinct, tolerant status line in the
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
# Container images — built, verified and published by CI, never from a workstation.
|
||||
#
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# WHY THIS FILE EXISTS (operator rule, 2026-10-29)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# "Container image creation, hosting and lifecycle must be part of CI/CD and the codebase, never a
|
||||
# one-off build/push from a workstation." Before this workflow, three images existed only as
|
||||
# workstation builds and NOTHING had ever been pushed to the Gitea registry: the Home-Network org's
|
||||
# package list was empty, platform/pm-agent/ pinned an all-zero digest and sat in ImagePullBackOff,
|
||||
# and conformance/versions.env carried an empty digest. Both build scripts said "publishing is an
|
||||
# operator action" — a step with no log, no recorded inputs and no verification anyone could read.
|
||||
#
|
||||
# WHAT IT DOES — the logic is in scripts/images.sh; this file only wires triggers and secrets.
|
||||
# pull_request -> build + VERIFY only. No push, and no registry credential is referenced.
|
||||
# push to main -> build + verify + PUBLISH to gitea.home.keithjackson.dev/home-network/<image>
|
||||
# dispatch -> same as a push when run on main; verify-only on any other ref.
|
||||
# Images: toolbox (infra/devcontainer), agent-runner (infra/agent, FROM toolbox) and
|
||||
# conformance-runner (conformance/incluster). Only images whose inputs changed are built.
|
||||
#
|
||||
# THE DIGEST LIFECYCLE — this workflow NEVER writes to git.
|
||||
# 1. publish pushes `sha-<commit>` (immutable) and `main` (moving), and prints the digest the
|
||||
# REGISTRY returns in the job summary.
|
||||
# 2. Renovate (renovate.yaml) follows the `main` tag of each image through the
|
||||
# `# renovate-digest:` markers next to every digest reference, and opens the bump PR.
|
||||
# 3. That PR goes through the gate like any other change, and Flux deploys it.
|
||||
# The bump PR is Renovate's and not this job's because a PR opened with the Actions token does not
|
||||
# trigger workflows, so it could never pass the REQUIRED gate check; Renovate's token is a real
|
||||
# user's.
|
||||
#
|
||||
# THE PUSH CREDENTIAL — `REGISTRY_PUSH_TOKEN`, a repository Actions secret, referenced BY NAME.
|
||||
# The automatic Actions token CANNOT be used. Gitea 1.24.3 source, verified:
|
||||
# * services/auth/basic.go (Basic.Verify) turns a task token into the pseudo-user
|
||||
# `gitea-actions` (ActionsUserID -2) and sets NO token scope;
|
||||
# * routers/api/packages/container/container.go (Authenticate, the /v2/token exchange every
|
||||
# docker client uses) then requires read:package / write:package / all scope and answers 401;
|
||||
# * services/context/package.go (determineAccessMode) has `// TODO: ActionUser permission check`
|
||||
# and no branch for that user, so even past login it would get at most READ on a public org.
|
||||
# So: a personal access token with scope `write:package`, minted by an account that has package
|
||||
# WRITE on the Home-Network org (org owner, or a team with the Packages unit at write). A dedicated
|
||||
# bot account in a packages-only team is the least-privilege choice. The username sent with it is
|
||||
# not checked by Gitea (basic.go resolves the owner from the token), so `vars.REGISTRY_PUSH_USER`
|
||||
# is optional and only labels the login.
|
||||
#
|
||||
# THE BUILDER — rootless BuildKit (operator decision on PR #81, 2026-10-29).
|
||||
# The act runner's job containers get NO Docker daemon (act_runner 0.2.11 forces docker_host "-"
|
||||
# for a tcp:// DinD, and gitea/act strips every bind when valid_volumes is empty — measured on
|
||||
# this PR's first run), and handing them the privileged DinD daemon would be root on the node for
|
||||
# anyone who can push a branch. So images are built by platform/buildkit/: a ROOTLESS buildkitd
|
||||
# (not privileged; uid 1000 in a user namespace), reached over mTLS at
|
||||
# buildkitd.buildkit.svc.cluster.local:1234 with a pinned `buildx` (remote driver). Its client
|
||||
# certificate reaches jobs as three Actions secrets — BUILDKIT_CA, BUILDKIT_CLIENT_CERT,
|
||||
# BUILDKIT_CLIENT_KEY — copied from cert-manager by infra/gitea/buildkit-client-secret.sh.
|
||||
# Pull requests get the client certificate (they must build) but never a registry credential.
|
||||
#
|
||||
# UNTIL platform/buildkit IS DEPLOYED (Flux applies it only after merge) THIS JOB FAILS, LOUDLY,
|
||||
# at its builder preflight. A skipped Gitea job reports `success`, so it is red on purpose.
|
||||
#
|
||||
# VERIFICATION is each Containerfile's `verify` stage, built before anything is pushed.
|
||||
#
|
||||
name: images
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- infra/devcontainer/Containerfile
|
||||
- infra/devcontainer/versions.env
|
||||
- infra/devcontainer/build-image.sh
|
||||
- infra/agent/**
|
||||
- conformance/incluster/Containerfile
|
||||
- conformance/incluster/conformance-clone
|
||||
- conformance/incluster/build-image.sh
|
||||
- .gitea/workflows/images.yaml
|
||||
- .gitea/workflows/scripts/images.sh
|
||||
- platform/buildkit/versions.env
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- infra/devcontainer/Containerfile
|
||||
- infra/devcontainer/versions.env
|
||||
- infra/devcontainer/build-image.sh
|
||||
- infra/agent/**
|
||||
- conformance/incluster/Containerfile
|
||||
- conformance/incluster/conformance-clone
|
||||
- conformance/incluster/build-image.sh
|
||||
- .gitea/workflows/images.yaml
|
||||
- .gitea/workflows/scripts/images.sh
|
||||
- platform/buildkit/versions.env
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
images:
|
||||
description: "Which images: 'all', or a space-separated subset of: toolbox agent-runner conformance-runner"
|
||||
required: false
|
||||
default: "all"
|
||||
|
||||
concurrency:
|
||||
# Never cancel a publish half-way: a pushed `sha-` tag with no matching `main` move is
|
||||
# recoverable (a rerun skips what is already published) but pointless to provoke.
|
||||
group: images-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
images:
|
||||
runs-on: ubuntu-latest
|
||||
# Measured on the PR that introduced this file — see its decisions entry. Generous because a
|
||||
# cold DinD store downloads the Fedora base and every pinned tool on each run.
|
||||
timeout-minutes: 75
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # the per-image change detection diffs against the base commit
|
||||
|
||||
# ── pull requests: build + verify, NO credential ───────────────────────────────────────
|
||||
- name: Build and verify (pull request — nothing is pushed)
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
MODE: verify
|
||||
IMAGES: auto
|
||||
PR_BASE_REF: ${{ github.base_ref }}
|
||||
# The BUILDER's client certificate (platform/buildkit). Not a registry credential:
|
||||
# a pull request can build, never push.
|
||||
BUILDKIT_CA: ${{ secrets.BUILDKIT_CA }}
|
||||
BUILDKIT_CLIENT_CERT: ${{ secrets.BUILDKIT_CLIENT_CERT }}
|
||||
BUILDKIT_CLIENT_KEY: ${{ secrets.BUILDKIT_CLIENT_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BASE_SHA="$(git merge-base "origin/${PR_BASE_REF}" HEAD)"
|
||||
export BASE_SHA
|
||||
./.gitea/workflows/scripts/images.sh
|
||||
|
||||
# ── non-main dispatch: verify only ──────────────────────────────────────────────────────
|
||||
- name: Build and verify (dispatch off main — nothing is pushed)
|
||||
if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main'
|
||||
env:
|
||||
MODE: verify
|
||||
IMAGES: ${{ inputs.images == 'all' && 'toolbox agent-runner conformance-runner' || inputs.images }}
|
||||
BUILDKIT_CA: ${{ secrets.BUILDKIT_CA }}
|
||||
BUILDKIT_CLIENT_CERT: ${{ secrets.BUILDKIT_CLIENT_CERT }}
|
||||
BUILDKIT_CLIENT_KEY: ${{ secrets.BUILDKIT_CLIENT_KEY }}
|
||||
run: ./.gitea/workflows/scripts/images.sh
|
||||
|
||||
# ── main: publish ───────────────────────────────────────────────────────────────────────
|
||||
- name: Preflight - the push token must exist, and its absence must be LOUD
|
||||
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
REGISTRY_PUSH_TOKEN: ${{ secrets.REGISTRY_PUSH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${REGISTRY_PUSH_TOKEN:-}" ]; then
|
||||
echo "::error::REGISTRY_PUSH_TOKEN is not set on this repository." >&2
|
||||
cat >&2 <<'MSG'
|
||||
|
||||
Images cannot be published without a registry credential, and this step fails rather
|
||||
than skipping ON PURPOSE: a skipped Gitea job reports `success`, so skipping here would
|
||||
make "nothing was published" look exactly like "published". That is how this repo went
|
||||
a month with images that existed only on one workstation.
|
||||
|
||||
WHY NOT THE BUILT-IN ACTIONS TOKEN: Gitea 1.24.3 gives it no package scope at the
|
||||
registry's /v2/token exchange (routers/api/packages/container/container.go,
|
||||
Authenticate) — see the header of .gitea/workflows/images.yaml.
|
||||
|
||||
TO FIX: as an account with package WRITE on the Home-Network org (ideally a dedicated
|
||||
bot in a packages-only team), create an access token with scope `write:package` and
|
||||
nothing else, then add it to this repository as the Actions secret
|
||||
REGISTRY_PUSH_TOKEN. Optionally set the Actions variable REGISTRY_PUSH_USER to that
|
||||
account's name. Do not commit the token anywhere. Then re-run this workflow
|
||||
(workflow_dispatch on main, images: all).
|
||||
MSG
|
||||
exit 1
|
||||
fi
|
||||
echo "REGISTRY_PUSH_TOKEN is present (value not printed)."
|
||||
|
||||
- name: Build, verify and publish (main)
|
||||
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
MODE: publish
|
||||
IMAGES: ${{ github.event_name == 'workflow_dispatch' && (inputs.images == 'all' && 'toolbox agent-runner conformance-runner' || inputs.images) || 'auto' }}
|
||||
BASE_SHA: ${{ github.event.before }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_PUSH_TOKEN }}
|
||||
REGISTRY_USER: ${{ vars.REGISTRY_PUSH_USER }}
|
||||
BUILDKIT_CA: ${{ secrets.BUILDKIT_CA }}
|
||||
BUILDKIT_CLIENT_CERT: ${{ secrets.BUILDKIT_CLIENT_CERT }}
|
||||
BUILDKIT_CLIENT_KEY: ${{ secrets.BUILDKIT_CLIENT_KEY }}
|
||||
run: ./.gitea/workflows/scripts/images.sh
|
||||
@@ -60,9 +60,9 @@ jobs:
|
||||
spec-immutability:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4 # the implementation repo
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # the implementation repo
|
||||
- name: Check out the read-only spec (sibling path)
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
repository: ${{ vars.SPEC_REPO_SLUG }} # e.g. Home-Network/home-network-k3s (set in Gitea)
|
||||
path: ../home-network-k3s
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [spec-immutability]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 } # full history so touched-requirement diff works
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh lint
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l0
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [l0]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [render-golden]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
run: make -C ci l1
|
||||
- name: Upload L1 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: l1-junit-coverage
|
||||
path: |
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
needs: [l1]
|
||||
if: ${{ vars.CLUSTER_AVAILABLE == 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh conformance
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
run: make -C ci conformance
|
||||
- name: Upload L2/L3 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: conformance-junit-coverage
|
||||
path: |
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
needs: [lint, l0, render-golden, l1, conformance]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Decide
|
||||
env:
|
||||
R_LINT: ${{ needs.lint.result }}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
# Renovate — the dependency/image update automation. SCHEDULED, self-hosted against this Gitea.
|
||||
#
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# WHY THIS FILE EXISTS: renovate.json HAS BEEN IN THIS REPO AND HAS NEVER ONCE RUN.
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Measured 2026-10-17: `renovate.json` is present, complete and thoughtfully written — it declares
|
||||
# managers, schedules, automerge classes and grouping. Nothing executes it. There is no Renovate
|
||||
# workflow (this is the first), no Renovate CronJob or Deployment in the cluster
|
||||
# (`kubectl get cronjob,deploy,job -A | grep -i renovate` -> nothing), and no bot account doing it
|
||||
# from outside. Meanwhile SEVEN files carry comments asserting "Renovate maintains this pin"
|
||||
# (platform/homepage/deployment.yaml:40, runbooks/serving/deployment.yaml:87,
|
||||
# platform/crafty/deployment.yaml:63, platform/vpn-ingress/daemonset.yaml:259,
|
||||
# platform/keycloak/postgres-cluster.yaml:75, apps/wave-0/nextcloud-db.yaml:302, and
|
||||
# platform/headlamp/helmrepository.yaml:2). Those comments have been false for the life of the file.
|
||||
#
|
||||
# The cost is measurable. With nothing bumping anything: the six per-app oauth2-proxy auth sidecars
|
||||
# sit on v7.8.1 while the platform oauth2-proxy in the SAME cluster runs v7.15.5; Keycloak (the IdP)
|
||||
# runs 26.0.6, a November-2024 release, 8 minors behind; Velero runs a RELEASE CANDIDATE
|
||||
# (velero-plugin-for-aws v1.14.3-rc.2) with a stable available; the CNPG Postgres image is 16.4,
|
||||
# missing every 16.5+ security release. A pin nothing maintains is not a pin, it is a fossil.
|
||||
#
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# WHY RENOVATE AND NOT FLUX IMAGE-AUTOMATION (the decision, and it reverses ADR-0009's split)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# ADR-0009 divided the work: Renovate owns Helm chart/dependency bumps, Flux image-automation owns
|
||||
# per-service container image TAGS, deliberately non-overlapping. The second half of that split
|
||||
# cannot be delivered on this repo's terms, for three measured reasons:
|
||||
#
|
||||
# 1. IT WOULD BYPASS THE MERGE GATE, WHICH CLAUDE.md §1 FORBIDS. ImageUpdateAutomation does not
|
||||
# open pull requests; it COMMITS AND PUSHES. Its default target is the source's own branch —
|
||||
# `main` — so it would write straight past the PR flow. It can be pointed at a side branch
|
||||
# instead, but then nothing opens the PR and bumps silently accumulate on a branch nobody
|
||||
# merges. Either way a human or a second bot has to close the loop.
|
||||
# 2. ON THIS CLUSTER IT CANNOT EVEN DO THAT. Measured 2026-10-17: `main` is protected with
|
||||
# `user_can_push: false`, and the `flux-system` GitRepository has NO `spec.secretRef` at all —
|
||||
# the source is anonymous and read-only. Enabling write-back means minting an in-cluster git
|
||||
# WRITE credential for the repository that defines the cluster, which is a materially larger
|
||||
# blast radius than a CI-scoped token.
|
||||
# 3. RENOVATE ALREADY OPENS PRs NATIVELY, so every change it proposes goes through the gate with
|
||||
# no extra machinery.
|
||||
#
|
||||
# So: Renovate owns BOTH halves, and Flux image-automation stays disabled. Note this is a change of
|
||||
# plan, not a change of state — image-automation has been commented out at
|
||||
# platform/kustomization.yaml:52 since it was authored, and the cluster has ZERO
|
||||
# ImageRepository/ImagePolicy/ImageUpdateAutomation objects (measured). Its two controllers ARE
|
||||
# installed and have idled for 43 days. Nothing is being torn down here; a path that was never
|
||||
# taken is being formally closed, and renovate.json's `docker`/`kubernetes` managers — disabled
|
||||
# purely to avoid fighting a controller that never ran — are switched on to fill the gap.
|
||||
#
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# THIS WORKFLOW MUST NOT BE ABLE TO SUCCEED SILENTLY
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# A SKIPPED Gitea job reports `success` (CLAUDE.md §2), so "green" is not evidence this ran. Two
|
||||
# guards:
|
||||
# * The preflight step FAILS HARD (exit 1) if RENOVATE_TOKEN is absent, rather than skipping.
|
||||
# A missing credential on a dependency bot is indistinguishable from "nothing to update", and
|
||||
# that is precisely how renovate.json stayed inert for months.
|
||||
# * The summary step records what ran into the job summary unconditionally.
|
||||
# What this still does NOT give you is an alert when the SCHEDULE stops firing. See the handoff
|
||||
# entry's "What I did NOT verify" — that gap is stated, not papered over.
|
||||
#
|
||||
# OPERATOR PREREQUISITE — ONE SECRET, NOT COMMITTED HERE:
|
||||
# `RENOVATE_TOKEN` — a Gitea access token for the bot account (`renovate-bot`) with FIVE scopes.
|
||||
# Each is required by a call Renovate 44.x makes against Gitea 1.24.3, and Gitea refuses a
|
||||
# token missing one with `403 token does not have at least one of required scope(s)`:
|
||||
# read:user GET /user (platform init: initPlatform)
|
||||
# read:organization GET /orgs/{owner}, /orgs/{o}/labels (initRepo -> isOrg, then org labels)
|
||||
# write:repository clone/push branches, /pulls, /statuses
|
||||
# write:issue /issues, /labels, /issues/{n}/labels (dependency dashboard, PR labels;
|
||||
# Gitea files all of these under the ISSUE scope, not repository)
|
||||
# read:package the Gitea container registry (RENOVATE_HOST_RULES below)
|
||||
# CORRECTED 2026-10-30. This list used to name only write:repository + read:user (+ read:package),
|
||||
# and the operator minted exactly that. Run 350 then failed at
|
||||
# `GET /api/v1/orgs/Home-Network 403 ... required=[read:organization], token scope=read:package,
|
||||
# write:repository,read:user`; write:issue would have been the next wall. The preflight below now
|
||||
# PROBES these scopes, so a short token fails in seconds with the missing scope named.
|
||||
# Set it as an Actions secret on this repository. Nothing in this repo mints,
|
||||
# stores or references its value; it is consumed by name only.
|
||||
# Until it exists, this workflow FAILS LOUDLY on every scheduled run. That is intended: a
|
||||
# visible red is the correct state for "automation configured but not credentialed".
|
||||
#
|
||||
# THE FIRST RUN WILL WANT TO OPEN A LOT OF PRs. ~30 dependencies are behind. renovate.json caps the
|
||||
# blast radius (`prConcurrentLimit: 5`, `prHourlyLimit: 2`), but run it manually with
|
||||
# `dry_run: true` FIRST and read the log before letting the schedule loose. That is what
|
||||
# workflow_dispatch's default is for.
|
||||
name: renovate
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 08:15 UTC daily. renovate.json ALSO carries `"schedule": ["after 3am and before 6am"]` with
|
||||
# `"timezone": "America/New_York"`, and THAT inner schedule is authoritative — Renovate re-checks
|
||||
# it and DECLINES TO OPEN PRs outside it. So this cron must fire INSIDE the local window, not
|
||||
# merely near it.
|
||||
#
|
||||
# WAS `15 3 * * *`, WHICH NEVER WOULD HAVE WORKED (fixed 2026-10-18, before the first scheduled
|
||||
# run). Gitea Actions cron is UTC, so 03:15 UTC is 23:15 the PREVIOUS EVENING in New York --
|
||||
# 3h45m BEFORE the window opens, not "slightly ahead" of it as the original comment claimed.
|
||||
# Every nightly run would have started, logged "not within schedule", opened zero PRs and
|
||||
# exited SUCCESSFULLY. A green tick forever, and no updates ever: the exact "failure mode is a
|
||||
# green tick" shape this repo keeps getting burned by, and indistinguishable from "nothing
|
||||
# needed updating".
|
||||
#
|
||||
# THE WINDOW IN UTC MOVES WITH US DST, and a cron cannot follow it:
|
||||
# EDT (UTC-4, Mar-Nov): local 03:00-06:00 == 07:00-10:00 UTC
|
||||
# EST (UTC-5, Nov-Mar): local 03:00-06:00 == 08:00-11:00 UTC
|
||||
# 08:15 UTC is the only quarter-hour comfortably inside BOTH (04:15 EDT / 03:15 EST), so the
|
||||
# schedule keeps working across both switchovers without an edit. Do not "tidy" this back to an
|
||||
# early-morning-UTC value without redoing this arithmetic.
|
||||
- cron: "15 8 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run (no PRs, no branches). Leave true for the first run."
|
||||
required: false
|
||||
default: "true"
|
||||
log_level:
|
||||
description: "Renovate LOG_LEVEL (info | debug)"
|
||||
required: false
|
||||
default: "info"
|
||||
|
||||
concurrency:
|
||||
# Never two Renovate runs at once: concurrent runs race on the same branches and can open
|
||||
# duplicate PRs. Not cancel-in-progress — a half-finished Renovate run can leave a branch pushed
|
||||
# with no PR, so let the first one finish.
|
||||
group: renovate
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
renovate:
|
||||
runs-on: ubuntu-latest
|
||||
# RENOVATE RUNS AS THE JOB CONTAINER, NOT THROUGH renovatebot/github-action. FIXED 2026-10-30.
|
||||
# That action does exactly one thing: `docker run ghcr.io/renovatebot/renovate:<version>`. On
|
||||
# this runner a job has NO Docker daemon, by decision (decisions/2026-10-29-image-pipeline.md:
|
||||
# act_runner sets container.docker_host to `-` when DOCKER_HOST is tcp://, and the DinD daemon
|
||||
# is privileged, so a job that could reach it would be root on the node). So every run of the
|
||||
# action died before Renovate started. Dispatch run 343 (2026-10-30), verbatim:
|
||||
# [command]/usr/bin/docker run --env LOG_LEVEL ... --env RENOVATE_TOKEN=*** ...
|
||||
# failed to connect to the docker API at unix:///var/run/docker.sock; ... no such file or directory
|
||||
# ::error::Error: The process '/usr/bin/docker' failed with exit code 1
|
||||
# No auth, scope, config or network question was ever reached. Making the Renovate image the
|
||||
# job container needs no daemon: act starts it through the runner's own DinD exactly as it
|
||||
# starts the default ubuntu image, then `exec`s each step inside it.
|
||||
#
|
||||
# PINNED here (was the action's `renovate-version: 44.167.0`). renovate.json5 is validated
|
||||
# against this exact tag — keep them in step. Renovate's github-actions manager reads
|
||||
# `.gitea/workflows` and extracts `container.image`, so this pin is still self-maintained.
|
||||
#
|
||||
# The image runs as uid 12021 (`ubuntu`), and that is fine under act: the step command files
|
||||
# ($GITHUB_STEP_SUMMARY etc.) are created mode 0666 (gitea/act v0.261.2 pkg/runner/step.go),
|
||||
# HOME comes from the image env, and no step writes the workspace. There is deliberately NO
|
||||
# actions/checkout: Renovate clones the repo itself from RENOVATE_ENDPOINT, and as uid 12021
|
||||
# checkout could not `git init` the root-owned workspace volume anyway.
|
||||
container:
|
||||
image: ghcr.io/renovatebot/renovate:44.167.0
|
||||
# Renovate scans every manager across the whole repo; 40m is generous for this repo's size and
|
||||
# well under a runner's patience. Without a timeout a wedged run holds the `renovate`
|
||||
# concurrency group and silently blocks every subsequent scheduled run.
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- name: Preflight - the token must exist, and its absence must be LOUD
|
||||
shell: bash
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${RENOVATE_TOKEN:-}" ]; then
|
||||
echo "::error::RENOVATE_TOKEN is not set on this repository." >&2
|
||||
cat >&2 <<'MSG'
|
||||
|
||||
Renovate cannot run without a Gitea access token, and this step fails rather than
|
||||
skipping ON PURPOSE: a skipped Gitea job reports `success`, so skipping here would make
|
||||
"dependency automation is not running" look exactly like "everything is up to date".
|
||||
That is the failure mode that let renovate.json sit inert in this repo for months while
|
||||
seven files claimed "Renovate maintains this pin".
|
||||
|
||||
TO FIX: create a Gitea access token for the bot account (renovate-bot) with scopes
|
||||
read:user, read:organization, write:repository, write:issue and read:package (the
|
||||
header of .gitea/workflows/renovate.yaml says which call needs which), then add it to
|
||||
this repository as the Actions secret RENOVATE_TOKEN. Do not commit it anywhere.
|
||||
MSG
|
||||
exit 1
|
||||
fi
|
||||
echo "RENOVATE_TOKEN is present (value not printed)."
|
||||
# Proves the step is running INSIDE the pinned Renovate image, not the default job image.
|
||||
echo "renovate $(renovate --version)"
|
||||
|
||||
- name: Preflight - the token has every scope Renovate needs
|
||||
shell: bash
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
# ADDED 2026-10-30. Renovate reports EVERY platform-init failure as the bare
|
||||
# "Authentication failure" (lib/modules/platform/gitea/index.ts initPlatform swallows the
|
||||
# HTTP error), and a later scope gap surfaces as "Repository has unknown error". Two
|
||||
# dispatches were spent learning one missing scope at a time. These probes are GETs only,
|
||||
# and they name the missing scope. The token never reaches argv: curl reads it from stdin
|
||||
# (`--config -`), so it is not in `ps` output or in the log.
|
||||
run: |
|
||||
set -uo pipefail
|
||||
GITEA=https://gitea.home.keithjackson.dev
|
||||
REPO=Home-Network/home-network-k3s-impl
|
||||
fail=0
|
||||
body=$(mktemp)
|
||||
probe() { # <scope it proves> <path> <auth: bearer|basic>
|
||||
local code
|
||||
if [ "$3" = basic ]; then
|
||||
code=$(printf 'user = "renovate:%s"\n' "$RENOVATE_TOKEN" \
|
||||
| curl -sS --config - -o "$body" -w '%{http_code}' "$GITEA$2") || code=000
|
||||
else
|
||||
code=$(printf 'header = "Authorization: Bearer %s"\n' "$RENOVATE_TOKEN" \
|
||||
| curl -sS --config - -o "$body" -w '%{http_code}' "$GITEA$2") || code=000
|
||||
fi
|
||||
if [ "$code" = 200 ]; then
|
||||
echo "ok $1 (GET ${2%%\?*} -> 200)"
|
||||
return
|
||||
fi
|
||||
fail=1
|
||||
echo "::error::scope $1: GET ${2%%\?*} -> $code"
|
||||
case "$code" in
|
||||
401) echo " 401 = Gitea rejected the token VALUE (wrong, expired or revoked)." ;;
|
||||
403) if grep -q 'Access denied' "$body"; then
|
||||
echo " 403 'Access denied' = a Cilium/Envoy policy denial, not Gitea."
|
||||
else
|
||||
echo " Gitea: $(jq -r '.message // empty' "$body" 2>/dev/null | head -c 300)"
|
||||
fi ;;
|
||||
*) echo " body: $(head -c 200 "$body")" ;;
|
||||
esac
|
||||
}
|
||||
probe read:user "/api/v1/user" bearer
|
||||
probe read:organization "/api/v1/orgs/${REPO%%/*}" bearer
|
||||
probe read:repository "/api/v1/repos/$REPO" bearer
|
||||
probe read:issue "/api/v1/repos/$REPO/labels?limit=1" bearer
|
||||
probe read:package "/v2/token?service=container_registry&scope=repository:home-network/toolbox:pull" basic
|
||||
# WRITE scopes cannot be proven by a GET. Gitea's scope-refusal message, however, lists
|
||||
# the token's scopes (names only, never the value): ask for an admin route this bot
|
||||
# cannot have, and read `token scope=` from the 403. Advisory if the format changes.
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$RENOVATE_TOKEN" \
|
||||
| curl -sS --config - -o "$body" "$GITEA/api/v1/admin/orgs?limit=1" || true
|
||||
scopes=$(jq -r '.message // empty' "$body" 2>/dev/null | sed -n 's/.*token scope=//p')
|
||||
if [ -n "$scopes" ]; then
|
||||
echo "token scopes (from Gitea): $scopes"
|
||||
for want in write:repository write:issue; do
|
||||
case ",$scopes," in
|
||||
*",$want,"*|*",all,"*) echo "ok $want" ;;
|
||||
*) echo "::error::scope $want is missing (token scope=$scopes)"; fail=1 ;;
|
||||
esac
|
||||
done
|
||||
else
|
||||
echo "::warning::could not read the token's scope list; write:repository and write:issue are UNVERIFIED"
|
||||
fi
|
||||
rm -f "$body"
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo "::error::RENOVATE_TOKEN failed the checks above (a 401 is the value, a 403 is a scope). Re-mint it for renovate-bot with read:user, read:organization, write:repository, write:issue, read:package and overwrite the secret."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Renovate
|
||||
shell: bash
|
||||
# The repo config is discovered by Renovate from the cloned repo (renovate.json5,
|
||||
# RENOVATE_REQUIRE_CONFIG=required). The action used to ALSO load it as the global config
|
||||
# via RENOVATE_CONFIG_FILE; that double-loading is gone. ci/lint.sh's `configurationFile:`
|
||||
# check therefore has nothing to resolve in this file, and passes.
|
||||
env:
|
||||
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||
# Self-hosted Gitea. `platform` + `endpoint` were the two values renovate.json's own
|
||||
# header said were "set at runtime by the operator's renovate runner" — this is that
|
||||
# runner, so they are set here rather than in the repo config, which keeps renovate.json
|
||||
# portable and free of this instance's URL.
|
||||
RENOVATE_PLATFORM: gitea
|
||||
RENOVATE_ENDPOINT: https://gitea.home.keithjackson.dev/api/v1
|
||||
RENOVATE_REPOSITORIES: Home-Network/home-network-k3s-impl
|
||||
# Renovate must never push to a protected branch; it works on its own branches and opens
|
||||
# PRs against this base. Stated explicitly rather than inherited, because this is the
|
||||
# property that keeps the whole mechanism inside CLAUDE.md §1's branch -> PR -> gate path.
|
||||
RENOVATE_BASE_BRANCHES: main
|
||||
RENOVATE_ONBOARDING: "false"
|
||||
RENOVATE_REQUIRE_CONFIG: "required"
|
||||
RENOVATE_DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == 'true' && 'full' || '' }}
|
||||
LOG_LEVEL: ${{ github.event_name == 'workflow_dispatch' && inputs.log_level || 'info' }}
|
||||
# ADDED 2026-10-29: registry read access for this repo's OWN images (toolbox,
|
||||
# agent-runner, conformance-runner — published by .gitea/workflows/images.yaml and
|
||||
# followed through `# renovate-digest:` markers). The Gitea registry answers 401 to
|
||||
# anonymous reads, and Renovate scopes the platform token to hostType `gitea`, which a
|
||||
# docker lookup never matches — so without this rule every digest lookup fails. Same
|
||||
# token; it therefore needs scope `read:package` in addition to the ones listed in the
|
||||
# preflight above. Gitea ignores the username for token auth (services/auth/basic.go).
|
||||
RENOVATE_HOST_RULES: '[{"hostType":"docker","matchHost":"gitea.home.keithjackson.dev","username":"renovate","password":"${{ secrets.RENOVATE_TOKEN }}"}]'
|
||||
# OPTIONAL, ADDED 2026-10-30. A github.com token (a fine-grained PAT with NO permissions,
|
||||
# public-repo read only) for lookups of GitHub-hosted dependencies. Without it Renovate
|
||||
# SKIPS them with `github-token-required` — measured in a local dry run: 17 deps, i.e.
|
||||
# every `uses: actions/*` here plus kubevirt/kubevirt, kubevirt/containerized-data-importer
|
||||
# and intel/intel-device-plugins-for-kubernetes. Unset is safe: the step below drops the
|
||||
# empty variable, and those deps simply stay unmaintained, with a WARN in the log.
|
||||
GITHUB_COM_TOKEN: ${{ secrets.RENOVATE_GITHUB_COM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# An expression that evaluates to '' still EXPORTS the variable, as an empty string.
|
||||
# Drop those so an unset value means "Renovate's default", never "set to empty".
|
||||
for v in RENOVATE_DRY_RUN GITHUB_COM_TOKEN; do
|
||||
[ -n "${!v:-}" ] || unset "$v"
|
||||
done
|
||||
# Through the image's entrypoint, as `docker run` would have gone: it sources the
|
||||
# containerbase env and initialises the bundled tools before exec'ing Renovate.
|
||||
exec renovate-entrypoint.sh renovate
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Renovate"
|
||||
echo
|
||||
echo "- trigger: \`${{ github.event_name }}\`"
|
||||
echo "- dry run: \`${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }}\`"
|
||||
echo "- outcome: \`${{ job.status }}\`"
|
||||
echo
|
||||
echo "Renovate owns BOTH Helm chart bumps and container image tags here; Flux"
|
||||
echo "image-automation is deliberately disabled (platform/kustomization.yaml)."
|
||||
echo "Every change it proposes arrives as a pull request and must pass the merge gate."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
Executable
+820
@@ -0,0 +1,820 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CVE triage bot (docs/17 "The triage bot", ADR-0027; R-VULN-05, R-VULN-07).
|
||||
|
||||
Runs in .gitea/workflows/cve-triage.yaml, once a day, before Renovate. Each run:
|
||||
1. reads report.json from the scanner's internal API;
|
||||
2. creates or updates the single `CVE Dashboard` issue (aggregates only);
|
||||
3. reads Renovate's Dependency Dashboard and open PRs, derives each image's remediation state;
|
||||
4. ticks `approve-branch` boxes on that dashboard, for packages matching an image with an
|
||||
unaccepted CRITICAL that has a fix;
|
||||
5. POSTs the heartbeat and the remediation states to the scanner.
|
||||
|
||||
WHAT IT MUST NEVER DO. Renovate is the only author of version changes (R-UPD-02): this script
|
||||
never edits a version, opens a bump PR, merges anything, or posts `/approve-override`. It never
|
||||
ticks a `rebase-branch` row. The dashboard issue carries no CVE id, package, image or workload,
|
||||
because the repository is readable without authentication, and no label the in-cluster PM acts
|
||||
on (platform/pm-agent/configmap.yaml PM_REQUIRE_LABEL = `agent`).
|
||||
|
||||
OUTPUT IS COUNTS ONLY. Job logs and the summary are as visible as the issue, so nothing here
|
||||
prints a report string. Every report value is untrusted: strings are validated, and only counts
|
||||
and timestamps that passed RFC 3339 validation ever reach markdown.
|
||||
|
||||
HOW A JOB CONTAINER REACHES THE SCANNER, AND WHAT IS NOT PROVEN
|
||||
---------------------------------------------------------------
|
||||
Jobs are containers on a per-job docker bridge network inside the DinD sidecar of the act-runner
|
||||
pod (apps/wave-1/gitea-actions.yaml). The runner config sets no `container.network` and dockerd no
|
||||
`--dns`, so name resolution is docker's embedded resolver forwarding to the resolvers of the
|
||||
dockerd container, which are that pod's: cluster DNS. Egress is masqueraded to the pod IP
|
||||
(measured 2026-10-29, platform/buildkit/networkpolicy.yaml header), so a ClusterIP is routed and
|
||||
policy-matched as the act-runner pod in gitea-actions. images.yaml relies on the same path to
|
||||
reach buildkitd.buildkit.svc.cluster.local, which is indirect evidence only.
|
||||
UNVERIFIED: in-cluster DNS from a DinD job container.
|
||||
UNVERIFIED: ClusterIP reachability from a DinD job container to vuln-scanner:8081.
|
||||
Hence VULN_API_FALLBACK_URL (the Service's pinned ClusterIP), used only when the name fails to
|
||||
resolve. Anything else, including refused and timed-out connections, fails the job: an
|
||||
unreachable API is a red run, never a skip.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Iterable, NamedTuple
|
||||
|
||||
DASHBOARD_TITLE = "CVE Dashboard"
|
||||
DASHBOARD_LABEL = "cve-dashboard"
|
||||
DASHBOARD_LABEL_COLOR = "b60205"
|
||||
# Labels the in-cluster PM dispatcher selects work by. Never applied to anything here.
|
||||
FORBIDDEN_LABELS = frozenset({"agent"})
|
||||
RENOVATE_DASHBOARD_TITLE = "Dependency Dashboard"
|
||||
|
||||
SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
|
||||
FINDING_STATUSES = ("open", "fix-available", "accepted")
|
||||
TARGET_STATES = ("scanned", "pending", "stale", "failed", "credential-required", "by-tag", "not-covered")
|
||||
TARGET_TYPES = ("image", "vm", "node")
|
||||
RUN_STATUSES = ("complete", "degraded")
|
||||
REMEDIATION_STATES = ("pr-open", "approval-requested", "no-update-identified", "not-tracked")
|
||||
|
||||
MAX_REPORT_BYTES = 128 * 1024 * 1024
|
||||
MAX_API_BYTES = 8 * 1024 * 1024
|
||||
MAX_FIELD_LEN = 4096
|
||||
MAX_IMAGE_LEN = 512
|
||||
MAX_ISSUE_BODY = 2 * 1024 * 1024
|
||||
MAX_DASHBOARD_ROWS = 2000
|
||||
MAX_DETECTED_DEPS = 20000
|
||||
MAX_REMEDIATION = 5000
|
||||
MAX_PAGES = 200
|
||||
PAGE_SIZE = 50
|
||||
REPORT_STALE_HOURS = 26
|
||||
DATABASE_STALE_HOURS = 48
|
||||
|
||||
HTTP_TIMEOUT = 60
|
||||
|
||||
RFC3339_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})")
|
||||
IMAGE_RE = re.compile(r"[a-z0-9]([a-z0-9._:/-]*[a-z0-9])?")
|
||||
HTTP_URL_RE = re.compile(r"https?://[A-Za-z0-9.-]+(:\d{1,5})?(/[A-Za-z0-9._~%/-]*)?")
|
||||
REPO_RE = re.compile(r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+")
|
||||
|
||||
# Renovate's own row shape (lib/workers/repository/dependency-dashboard.ts, getListItem). The kind
|
||||
# is captured whole so `approvePr-branch` can never read as `approve-branch`.
|
||||
ROW_RE = re.compile(r" - \[(?P<mark>[ xX])\] <!-- (?P<kind>[A-Za-z]+)-branch=(?P<branch>\S+) -->(?P<rest>.*)")
|
||||
LINKED_TITLE_RE = re.compile(r"\[(?P<title>.*)\]\((?P<target>[^()\s]*)\)(?P<tail>.*)")
|
||||
PR_LINK_RE = re.compile(r"(?:^|/)pulls?/(\d{1,9})$")
|
||||
PACKAGE_LIST_RE = re.compile(r"\s\(((?:`[^`\s]+`(?:, )?)+)\)\s*$")
|
||||
TITLE_PACKAGE_RE = re.compile(r"(?:^|\s)(?:Update|Pin)\s+(\S+)\s+Docker\s+(?:tag|digest)\b")
|
||||
SUMMARY_RE = re.compile(r"<details><summary>([^<]*?)(?: \(\d+\))?</summary>")
|
||||
DETECTED_RE = re.compile(r" - `(?P<name>[^\s`]+)(?: [^`]*)?`(?P<updates>\s*→ \[Updates:.*\])?\s*")
|
||||
|
||||
# Detected-dependency managers whose names are charts or actions, never container images.
|
||||
NON_IMAGE_MANAGERS = frozenset({"flux", "helmv3", "github-actions"})
|
||||
|
||||
|
||||
class BotError(Exception):
|
||||
"""A failure that ends the run. The message must be safe to print: no report text, no secret."""
|
||||
|
||||
|
||||
class DnsFailure(Exception):
|
||||
"""The hostname did not resolve. The only condition that selects the fallback address."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- pure: normalisation + report
|
||||
|
||||
|
||||
def normalise_image(value: object) -> str:
|
||||
"""Canonical form for exact comparison, or "" when the value is not an image reference.
|
||||
|
||||
Lowercase; drop digest and tag; drop the Docker Hub registry and the `library/` namespace.
|
||||
`library/` is only Hub's implicit namespace, so ghcr.io/library/x keeps it. Comparison of the
|
||||
result is always string equality, never a substring test.
|
||||
"""
|
||||
if not isinstance(value, str) or not value or len(value) > MAX_IMAGE_LEN:
|
||||
return ""
|
||||
name = value.strip().lower().split("@", 1)[0]
|
||||
head, sep, last = name.rpartition("/")
|
||||
name = f"{head}{sep}{last.split(':', 1)[0]}"
|
||||
for hub in ("docker.io/", "index.docker.io/", "registry-1.docker.io/"):
|
||||
if name.startswith(hub):
|
||||
name = name[len(hub):]
|
||||
break
|
||||
if name.startswith("library/"):
|
||||
name = name[len("library/"):]
|
||||
return name if IMAGE_RE.fullmatch(name) else ""
|
||||
|
||||
|
||||
def parse_timestamp(value: object, field: str, nullable: bool = False) -> datetime | None:
|
||||
if value is None and nullable:
|
||||
return None
|
||||
if not isinstance(value, str) or not RFC3339_RE.fullmatch(value):
|
||||
raise BotError(f"report: {field} is not an RFC 3339 timestamp")
|
||||
text = re.sub(r"\.\d+", "", value).replace("Z", "+00:00")
|
||||
try:
|
||||
return datetime.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise BotError(f"report: {field} is not a valid timestamp") from exc
|
||||
|
||||
|
||||
def _object(value: object, field: str) -> dict:
|
||||
if not isinstance(value, dict):
|
||||
raise BotError(f"report: {field} is not an object")
|
||||
return value
|
||||
|
||||
|
||||
def _array(value: object, field: str) -> list:
|
||||
if not isinstance(value, list):
|
||||
raise BotError(f"report: {field} is not an array")
|
||||
return value
|
||||
|
||||
|
||||
def _string(value: object, field: str, nullable: bool = False) -> str | None:
|
||||
if value is None and nullable:
|
||||
return None
|
||||
if not isinstance(value, str) or len(value) > MAX_FIELD_LEN:
|
||||
raise BotError(f"report: {field} is not a bounded string")
|
||||
return value
|
||||
|
||||
|
||||
def _member(value: object, allowed: Iterable[str], field: str) -> str:
|
||||
if not isinstance(value, str) or value not in allowed:
|
||||
raise BotError(f"report: {field} has a value outside its enumeration")
|
||||
return value
|
||||
|
||||
|
||||
def parse_report(raw: bytes) -> dict:
|
||||
"""Decode and validate report.json into the only shape the rest of this file reads."""
|
||||
try:
|
||||
doc = json.loads(raw)
|
||||
except (ValueError, RecursionError) as exc:
|
||||
raise BotError("report: body is not valid JSON") from exc
|
||||
doc = _object(doc, "document")
|
||||
if doc.get("schemaVersion") != 1 or isinstance(doc.get("schemaVersion"), bool):
|
||||
raise BotError("report: unsupported schemaVersion")
|
||||
parse_timestamp(doc.get("generatedAt"), "generatedAt")
|
||||
database = _object(doc.get("database"), "database")
|
||||
run = _object(doc.get("run"), "run")
|
||||
parse_timestamp(database.get("updatedAt"), "database.updatedAt", nullable=True)
|
||||
parse_timestamp(run.get("startedAt"), "run.startedAt", nullable=True)
|
||||
parse_timestamp(run.get("finishedAt"), "run.finishedAt", nullable=True)
|
||||
if run.get("status") is not None:
|
||||
_member(run.get("status"), RUN_STATUSES, "run.status")
|
||||
|
||||
targets = []
|
||||
known_ids = {}
|
||||
for entry in _array(doc.get("targets"), "targets"):
|
||||
entry = _object(entry, "target")
|
||||
target_id = _string(entry.get("id"), "target.id")
|
||||
kind = _member(entry.get("type"), TARGET_TYPES, "target.type")
|
||||
state = _member(entry.get("state"), TARGET_STATES, "target.state")
|
||||
image = _string(entry.get("image"), "target.image", nullable=True)
|
||||
if kind == "image" and not normalise_image(image):
|
||||
raise BotError("report: an image target has an invalid image reference")
|
||||
known_ids[target_id] = image if kind == "image" else None
|
||||
targets.append({"id": target_id, "type": kind, "state": state, "image": image})
|
||||
|
||||
findings = []
|
||||
for entry in _array(doc.get("findings"), "findings"):
|
||||
entry = _object(entry, "finding")
|
||||
target_id = _string(entry.get("targetId"), "finding.targetId")
|
||||
if target_id not in known_ids:
|
||||
raise BotError("report: a finding references an unknown target")
|
||||
findings.append({
|
||||
"targetId": target_id,
|
||||
"image": known_ids[target_id],
|
||||
"severity": _member(entry.get("severity"), SEVERITIES, "finding.severity"),
|
||||
"status": _member(entry.get("status"), FINDING_STATUSES, "finding.status"),
|
||||
"fixedVersion": _string(entry.get("fixedVersion"), "finding.fixedVersion", nullable=True),
|
||||
})
|
||||
return {
|
||||
"generatedAt": doc["generatedAt"],
|
||||
"database": {"updatedAt": database.get("updatedAt")},
|
||||
"run": {"finishedAt": run.get("finishedAt"), "status": run.get("status")},
|
||||
"targets": targets,
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def qualifies(finding: dict) -> bool:
|
||||
"""An unaccepted CRITICAL with a fix: the only thing that may cause a tick."""
|
||||
return (
|
||||
finding["severity"] == "CRITICAL"
|
||||
and finding["status"] == "fix-available"
|
||||
and bool(finding["fixedVersion"])
|
||||
)
|
||||
|
||||
|
||||
def qualifying_images(report: dict) -> set[str]:
|
||||
return {
|
||||
norm
|
||||
for finding in report["findings"]
|
||||
if qualifies(finding) and (norm := normalise_image(finding["image"]))
|
||||
}
|
||||
|
||||
|
||||
def aggregate(report: dict) -> dict:
|
||||
severity = {sev: dict.fromkeys(FINDING_STATUSES, 0) for sev in SEVERITIES}
|
||||
for finding in report["findings"]:
|
||||
severity[finding["severity"]][finding["status"]] += 1
|
||||
coverage = dict.fromkeys(TARGET_STATES, 0)
|
||||
for target in report["targets"]:
|
||||
coverage[target["state"]] += 1
|
||||
return {
|
||||
"severity": severity,
|
||||
"coverage": coverage,
|
||||
"targets_total": len(report["targets"]),
|
||||
"critical_with_fix": sum(1 for f in report["findings"] if qualifies(f)),
|
||||
"generatedAt": report["generatedAt"],
|
||||
"databaseUpdatedAt": report["database"]["updatedAt"],
|
||||
"runFinishedAt": report["run"]["finishedAt"],
|
||||
"runStatus": report["run"]["status"],
|
||||
}
|
||||
|
||||
|
||||
def render_dashboard(agg: dict, page_url: str, now: datetime) -> str:
|
||||
"""The CVE Dashboard issue body: a fixed template over counts and validated timestamps."""
|
||||
if not HTTP_URL_RE.fullmatch(page_url):
|
||||
raise BotError("config: VULN_PAGE_URL is not a plain http(s) URL")
|
||||
warnings = []
|
||||
generated = parse_timestamp(agg["generatedAt"], "generatedAt")
|
||||
if now - generated > timedelta(hours=REPORT_STALE_HOURS):
|
||||
warnings.append(f"The report is older than {REPORT_STALE_HOURS} hours.")
|
||||
updated = parse_timestamp(agg["databaseUpdatedAt"], "database.updatedAt", nullable=True)
|
||||
if updated is None or now - updated > timedelta(hours=DATABASE_STALE_HOURS):
|
||||
warnings.append(f"The vulnerability database is older than {DATABASE_STALE_HOURS} hours or unknown.")
|
||||
if agg["runStatus"] != "complete":
|
||||
warnings.append("The latest scan run did not complete cleanly.")
|
||||
|
||||
lines = [
|
||||
"Maintained by the CVE triage bot, once a day. **Counts only**: this repository is readable",
|
||||
"without authentication, so which CVE lands where is only on the",
|
||||
f"[CVE page]({page_url}) (SSO, platform-admins).",
|
||||
"",
|
||||
]
|
||||
lines += [f"> **WARNING:** {text}" for text in warnings]
|
||||
if warnings:
|
||||
lines.append("")
|
||||
lines += [
|
||||
"## Freshness",
|
||||
"",
|
||||
"| | |",
|
||||
"|---|---|",
|
||||
f"| Report generated | {agg['generatedAt']} |",
|
||||
f"| Database updated | {agg['databaseUpdatedAt'] or 'unknown'} |",
|
||||
f"| Last run finished | {agg['runFinishedAt'] or 'none yet'} |",
|
||||
f"| Last run status | {agg['runStatus'] or 'none yet'} |",
|
||||
f"| Dashboard rendered | {now.strftime('%Y-%m-%dT%H:%M:%SZ')} |",
|
||||
"",
|
||||
"## Findings by severity",
|
||||
"",
|
||||
"| Severity | Open | Fix available | Accepted |",
|
||||
"|---|---:|---:|---:|",
|
||||
]
|
||||
for sev in SEVERITIES:
|
||||
row = agg["severity"][sev]
|
||||
lines.append(f"| {sev} | {row['open']} | {row['fix-available']} | {row['accepted']} |")
|
||||
lines += [
|
||||
"",
|
||||
f"Unaccepted CRITICAL with a fix: **{agg['critical_with_fix']}**",
|
||||
"",
|
||||
"## Coverage",
|
||||
"",
|
||||
f"{agg['targets_total']} targets. A count in any state other than `scanned` is something"
|
||||
" that was not freshly examined, and is not a clean result.",
|
||||
"",
|
||||
"| State | Targets |",
|
||||
"|---|---:|",
|
||||
]
|
||||
lines += [f"| {state} | {agg['coverage'][state]} |" for state in TARGET_STATES]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- pure: Renovate's dashboard
|
||||
|
||||
|
||||
class Row(NamedTuple):
|
||||
kind: str
|
||||
mark: str
|
||||
branch: str
|
||||
packages: tuple[str, ...]
|
||||
pr_number: int | None
|
||||
offset: int # offset in the body of the first character of the line
|
||||
|
||||
|
||||
def _title_package(title: str) -> str:
|
||||
match = TITLE_PACKAGE_RE.search(title)
|
||||
return normalise_image(match.group(1)) if match else ""
|
||||
|
||||
|
||||
def _row_details(rest: str) -> tuple[tuple[str, ...], int | None]:
|
||||
"""Packages and PR number of a row. No recognisable package means no packages, so no tick."""
|
||||
rest = rest.rstrip("\r")
|
||||
number = None
|
||||
title = rest
|
||||
tail = rest
|
||||
linked = LINKED_TITLE_RE.fullmatch(rest)
|
||||
if linked:
|
||||
title, tail = linked.group("title"), linked.group("tail")
|
||||
link = PR_LINK_RE.search(linked.group("target"))
|
||||
number = int(link.group(1)) if link else None
|
||||
listed = PACKAGE_LIST_RE.search(tail)
|
||||
if listed:
|
||||
names = re.findall(r"`([^`]+)`", listed.group(1))
|
||||
packages = tuple(n for n in map(normalise_image, names) if n)
|
||||
else:
|
||||
package = _title_package(title)
|
||||
packages = (package,) if package else ()
|
||||
return packages, number
|
||||
|
||||
|
||||
def _is_detected_heading(line: str) -> bool:
|
||||
return line.startswith("## ") and line[3:].strip().lower().startswith("detected dependencies")
|
||||
|
||||
|
||||
def parse_rows(body: str) -> list[Row]:
|
||||
"""Checkbox rows above `## Detected Dependencies`. Only a whole-line match counts."""
|
||||
if len(body) > MAX_ISSUE_BODY:
|
||||
raise BotError("renovate dashboard: body exceeds its bound")
|
||||
rows: list[Row] = []
|
||||
offset = 0
|
||||
for line in body.split("\n"):
|
||||
if _is_detected_heading(line):
|
||||
break
|
||||
match = ROW_RE.fullmatch(line.rstrip("\r"))
|
||||
if match:
|
||||
packages, number = _row_details(match.group("rest"))
|
||||
rows.append(Row(match.group("kind"), match.group("mark"), match.group("branch"),
|
||||
packages, number, offset))
|
||||
if len(rows) > MAX_DASHBOARD_ROWS:
|
||||
raise BotError("renovate dashboard: too many rows")
|
||||
offset += len(line) + 1
|
||||
return rows
|
||||
|
||||
|
||||
def parse_detected(body: str) -> dict[str, bool]:
|
||||
"""{normalised depName: has a pending update} from `## Detected Dependencies`."""
|
||||
tracked: dict[str, bool] = {}
|
||||
lines = body.split("\n")
|
||||
in_section = False
|
||||
manager = ""
|
||||
count = 0
|
||||
for index, raw in enumerate(lines):
|
||||
line = raw.rstrip("\r")
|
||||
if line.startswith("## "):
|
||||
in_section = _is_detected_heading(line)
|
||||
continue
|
||||
if not in_section:
|
||||
continue
|
||||
summary = SUMMARY_RE.fullmatch(line)
|
||||
if summary:
|
||||
# A manager heading is the summary directly followed by <blockquote>; a file heading
|
||||
# is not.
|
||||
if index + 1 < len(lines) and lines[index + 1].strip() == "<blockquote>":
|
||||
manager = summary.group(1)
|
||||
continue
|
||||
match = DETECTED_RE.fullmatch(line)
|
||||
if not match or manager in NON_IMAGE_MANAGERS:
|
||||
continue
|
||||
count += 1
|
||||
if count > MAX_DETECTED_DEPS:
|
||||
raise BotError("renovate dashboard: too many detected dependencies")
|
||||
norm = normalise_image(match.group("name"))
|
||||
if norm:
|
||||
tracked[norm] = tracked.get(norm, False) or bool(match.group("updates"))
|
||||
return tracked
|
||||
|
||||
|
||||
def row_matches(row: Row, qualifying: set[str]) -> bool:
|
||||
return any(package in qualifying for package in row.packages)
|
||||
|
||||
|
||||
def select_ticks(rows: list[Row], qualifying: set[str]) -> list[Row]:
|
||||
return [
|
||||
row for row in rows
|
||||
if row.kind == "approve" and row.mark == " " and row_matches(row, qualifying)
|
||||
]
|
||||
|
||||
|
||||
def apply_ticks(body: str, ticks: list[Row]) -> str:
|
||||
"""Replace only the `[ ]` of each row, re-verifying the row where it stands.
|
||||
|
||||
Every byte other than the three of each marker is carried over unchanged.
|
||||
"""
|
||||
pieces = []
|
||||
cursor = 0
|
||||
for row in sorted(ticks, key=lambda r: r.offset):
|
||||
expected = f" - [ ] <!-- approve-branch={row.branch} -->"
|
||||
if row.kind != "approve" or not body.startswith(expected, row.offset):
|
||||
raise BotError("renovate dashboard: a row changed between parse and write")
|
||||
marker = row.offset + len(" - ")
|
||||
pieces += [body[cursor:marker], "[x]"]
|
||||
cursor = marker + len("[ ]")
|
||||
pieces.append(body[cursor:])
|
||||
return "".join(pieces)
|
||||
|
||||
|
||||
def derive_remediation(
|
||||
images: Iterable[str],
|
||||
rows: list[Row],
|
||||
open_prs: list[dict],
|
||||
tracked: dict[str, bool],
|
||||
pr_url: Callable[[int], str],
|
||||
now_text: str,
|
||||
) -> list[dict]:
|
||||
"""One entry per report image.
|
||||
|
||||
pr-open: an open-PR row or an open Renovate PR for that package. approval-requested: an
|
||||
approve row. no-update-identified: Renovate tracks it and lists no update. not-tracked:
|
||||
everything else, which deliberately includes "Renovate knows an update but no PR or approval
|
||||
row exists" (awaiting schedule, rate limited): saying no update was identified would be
|
||||
false, and the enumeration has no closer value, so it lands on the state that draws attention.
|
||||
"""
|
||||
by_branch = {pr["branch"]: pr["number"] for pr in open_prs}
|
||||
result = []
|
||||
for image in sorted(set(images)):
|
||||
norm = normalise_image(image)
|
||||
state = "not-tracked"
|
||||
url = None
|
||||
rebase = next((r for r in rows if r.kind == "rebase" and norm in r.packages), None)
|
||||
pr = next((p for p in open_prs if p["package"] == norm), None)
|
||||
if rebase is not None or pr is not None:
|
||||
state = "pr-open"
|
||||
number = None
|
||||
if rebase is not None:
|
||||
number = rebase.pr_number or by_branch.get(rebase.branch)
|
||||
if number is None and pr is not None:
|
||||
number = pr["number"]
|
||||
url = pr_url(number) if number is not None else None
|
||||
elif any(r.kind == "approve" and norm in r.packages for r in rows):
|
||||
state = "approval-requested"
|
||||
elif norm in tracked and not tracked[norm]:
|
||||
state = "no-update-identified"
|
||||
result.append({"image": image, "state": state, "url": url, "updatedAt": now_text})
|
||||
return result
|
||||
|
||||
|
||||
def count_states(remediation: list[dict]) -> dict[str, int]:
|
||||
counts = dict.fromkeys(REMEDIATION_STATES, 0)
|
||||
for entry in remediation:
|
||||
counts[entry["state"]] += 1
|
||||
return counts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- I/O
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""A redirect would carry the Authorization header to wherever it points."""
|
||||
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
class UrllibTransport:
|
||||
"""(method, url, headers, body, max_bytes) -> (status, bytes); never uses a proxy."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect())
|
||||
|
||||
def __call__(self, method: str, url: str, headers: dict, body: bytes | None, max_bytes: int):
|
||||
request = urllib.request.Request(url, data=body, method=method, headers=headers)
|
||||
try:
|
||||
with self._opener.open(request, timeout=HTTP_TIMEOUT) as response:
|
||||
data = response.read(max_bytes + 1)
|
||||
return response.status, data
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, b""
|
||||
except urllib.error.URLError as exc:
|
||||
if isinstance(exc.reason, socket.gaierror):
|
||||
raise DnsFailure() from exc
|
||||
raise BotError(f"{method} {urllib.parse.urlsplit(url).path}: {type(exc.reason).__name__}") from exc
|
||||
except (OSError, http.client.HTTPException) as exc:
|
||||
raise BotError(f"{method} {urllib.parse.urlsplit(url).path}: {type(exc).__name__}") from exc
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
def __init__(self, base_url: str, repo: str, token: str, transport) -> None:
|
||||
self._api = f"{base_url.rstrip('/')}/api/v1/repos/{repo}"
|
||||
self._web = f"{base_url.rstrip('/')}/{repo}"
|
||||
self._token = token
|
||||
self._transport = transport
|
||||
|
||||
def _call(self, method: str, path: str, query: dict | None = None, payload=None, expect=(200,)):
|
||||
url = self._api + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
||||
headers = {"Authorization": f"token {self._token}", "Accept": "application/json"}
|
||||
data = None
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
try:
|
||||
status, raw = self._transport(method, url, headers, data, MAX_API_BYTES)
|
||||
except DnsFailure as exc:
|
||||
raise BotError(f"gitea: {method} {path}: name resolution failed") from exc
|
||||
if status not in expect:
|
||||
raise BotError(f"gitea: {method} {path} -> HTTP {status}")
|
||||
if len(raw) > MAX_API_BYTES:
|
||||
raise BotError(f"gitea: {method} {path}: response exceeds its bound")
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError as exc:
|
||||
raise BotError(f"gitea: {method} {path}: response is not JSON") from exc
|
||||
|
||||
def _list(self, path: str, query: dict | None = None) -> list:
|
||||
items: list = []
|
||||
for page in range(1, MAX_PAGES + 1):
|
||||
batch = self._call("GET", path, {**(query or {}), "limit": PAGE_SIZE, "page": page})
|
||||
if not isinstance(batch, list):
|
||||
raise BotError(f"gitea: GET {path}: expected a list")
|
||||
items += batch
|
||||
if len(batch) < PAGE_SIZE:
|
||||
return items
|
||||
raise BotError(f"gitea: GET {path}: more than {MAX_PAGES} pages")
|
||||
|
||||
def pr_url(self, number: int) -> str:
|
||||
return f"{self._web}/pulls/{int(number)}"
|
||||
|
||||
def labels(self) -> list:
|
||||
return self._list("/labels")
|
||||
|
||||
def create_label(self, name: str, color: str) -> dict:
|
||||
return self._call("POST", "/labels", payload={"name": name, "color": color}, expect=(201,))
|
||||
|
||||
def issues_with_label(self, label: str) -> list:
|
||||
return self._list("/issues", {"state": "all", "type": "issues", "labels": label})
|
||||
|
||||
def issues_matching(self, text: str) -> list:
|
||||
return self._list("/issues", {"state": "all", "type": "issues", "q": text})
|
||||
|
||||
def create_issue(self, title: str, body: str, label_id: int) -> dict:
|
||||
return self._call("POST", "/issues", payload={"title": title, "body": body, "labels": [label_id]}, expect=(201,))
|
||||
|
||||
def update_issue(self, number: int, fields: dict) -> None:
|
||||
self._call("PATCH", f"/issues/{int(number)}", payload=fields, expect=(200, 201))
|
||||
|
||||
def add_issue_label(self, number: int, label_id: int) -> None:
|
||||
self._call("POST", f"/issues/{int(number)}/labels", payload={"labels": [label_id]}, expect=(200,))
|
||||
|
||||
def get_issue(self, number: int) -> dict:
|
||||
issue = self._call("GET", f"/issues/{int(number)}")
|
||||
if not isinstance(issue, dict):
|
||||
raise BotError("gitea: issue response is not an object")
|
||||
return issue
|
||||
|
||||
def open_renovate_prs(self) -> list[dict]:
|
||||
result = []
|
||||
for pr in self._list("/pulls", {"state": "open"}):
|
||||
head = pr.get("head") if isinstance(pr, dict) else None
|
||||
ref = head.get("ref") if isinstance(head, dict) else None
|
||||
number = pr.get("number") if isinstance(pr, dict) else None
|
||||
title = pr.get("title") if isinstance(pr, dict) else None
|
||||
if not isinstance(ref, str) or not isinstance(number, int) or isinstance(number, bool):
|
||||
raise BotError("gitea: a pull request has an unexpected shape")
|
||||
if ref.startswith("renovate/"):
|
||||
package = _title_package(title) if isinstance(title, str) else ""
|
||||
result.append({"number": number, "branch": ref, "package": package})
|
||||
return result
|
||||
|
||||
|
||||
class ScannerClient:
|
||||
"""The scanner's internal API. Falls back to the pinned ClusterIP on a DNS failure only."""
|
||||
|
||||
def __init__(self, primary: str, fallback: str, transport) -> None:
|
||||
self._bases = [base.rstrip("/") for base in (primary, fallback) if base]
|
||||
self._transport = transport
|
||||
self.used_fallback = False
|
||||
|
||||
def _call(self, method: str, path: str, body: bytes | None, headers: dict, max_bytes: int):
|
||||
last = len(self._bases) - 1
|
||||
for index, base in enumerate(self._bases):
|
||||
try:
|
||||
result = self._transport(method, base + path, headers, body, max_bytes)
|
||||
except DnsFailure as exc:
|
||||
if index == last:
|
||||
raise BotError(f"scanner: {method} {path}: name resolution failed") from exc
|
||||
continue
|
||||
if index:
|
||||
self.used_fallback = True
|
||||
self._bases = self._bases[index:]
|
||||
return result
|
||||
raise BotError("scanner: no address configured")
|
||||
|
||||
def report(self) -> dict:
|
||||
status, raw = self._call("GET", "/report.json", None, {"Accept": "application/json"}, MAX_REPORT_BYTES)
|
||||
if status != 200:
|
||||
raise BotError(f"scanner: GET /report.json -> HTTP {status}")
|
||||
if len(raw) > MAX_REPORT_BYTES:
|
||||
raise BotError("scanner: report exceeds its bound")
|
||||
return parse_report(raw)
|
||||
|
||||
def triage(self, heartbeat: str, remediation: list[dict]) -> None:
|
||||
if len(remediation) > MAX_REMEDIATION:
|
||||
raise BotError("scanner: more remediation entries than POST /triage accepts")
|
||||
body = json.dumps({"heartbeatAt": heartbeat, "remediation": remediation}).encode()
|
||||
status, _ = self._call("POST", "/triage", body, {"Content-Type": "application/json"}, 4096)
|
||||
if status != 204:
|
||||
raise BotError(f"scanner: POST /triage -> HTTP {status}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- orchestration
|
||||
|
||||
|
||||
def _label_names(issue: dict) -> set:
|
||||
return {label.get("name") for label in (issue.get("labels") or []) if isinstance(label, dict)}
|
||||
|
||||
|
||||
def find_dashboard_issues(gitea: GiteaClient) -> list[dict]:
|
||||
"""Every issue that is the dashboard: by label, and by exact title in case a label was removed."""
|
||||
found: dict[int, dict] = {}
|
||||
for issue in gitea.issues_with_label(DASHBOARD_LABEL) + gitea.issues_matching(DASHBOARD_TITLE):
|
||||
if (
|
||||
isinstance(issue, dict)
|
||||
and issue.get("title") == DASHBOARD_TITLE
|
||||
and not issue.get("pull_request")
|
||||
and isinstance(issue.get("number"), int)
|
||||
):
|
||||
found[issue["number"]] = issue
|
||||
return list(found.values())
|
||||
|
||||
|
||||
def sync_dashboard_issue(gitea: GiteaClient, body: str, dry_run: bool) -> dict:
|
||||
"""Create or update the one dashboard issue. Returns counts of what it did or would do."""
|
||||
if DASHBOARD_LABEL in FORBIDDEN_LABELS:
|
||||
raise BotError("config: the dashboard label is one the PM agent acts on")
|
||||
plan = {"label_created": 0, "issue_created": 0, "issue_updated": 0, "issue_reopened": 0}
|
||||
existing = find_dashboard_issues(gitea)
|
||||
if len(existing) > 1:
|
||||
# Closing an issue on a guess is worse than a red run naming the numbers to look at.
|
||||
numbers = sorted(issue["number"] for issue in existing)
|
||||
raise BotError(f"dashboard: {len(numbers)} issues titled {DASHBOARD_TITLE!r}: {numbers}")
|
||||
|
||||
label = next((x for x in gitea.labels() if isinstance(x, dict) and x.get("name") == DASHBOARD_LABEL), None)
|
||||
if label is None:
|
||||
plan["label_created"] = 1
|
||||
if not dry_run:
|
||||
label = gitea.create_label(DASHBOARD_LABEL, DASHBOARD_LABEL_COLOR)
|
||||
|
||||
if not existing:
|
||||
plan["issue_created"] = 1
|
||||
if not dry_run:
|
||||
created = gitea.create_issue(DASHBOARD_TITLE, body, label["id"])
|
||||
if not isinstance(created, dict) or not isinstance(created.get("number"), int):
|
||||
raise BotError("gitea: issue creation returned an unexpected shape")
|
||||
return plan
|
||||
|
||||
issue = existing[0]
|
||||
fields: dict = {}
|
||||
if issue.get("body") != body:
|
||||
fields["body"] = body
|
||||
if issue.get("state") != "open":
|
||||
fields["state"] = "open"
|
||||
plan["issue_reopened"] = 1
|
||||
if fields:
|
||||
plan["issue_updated"] = 1
|
||||
if not dry_run:
|
||||
if DASHBOARD_LABEL not in _label_names(issue):
|
||||
gitea.add_issue_label(issue["number"], label["id"])
|
||||
if fields:
|
||||
gitea.update_issue(issue["number"], fields)
|
||||
return plan
|
||||
|
||||
|
||||
def _renovate_issue(gitea: GiteaClient, number: int) -> dict:
|
||||
issue = gitea.get_issue(number)
|
||||
body = issue.get("body")
|
||||
if issue.get("title") != RENOVATE_DASHBOARD_TITLE or issue.get("state") != "open" or not isinstance(body, str):
|
||||
raise BotError(f"renovate dashboard: issue {number} is not the open Dependency Dashboard")
|
||||
return issue
|
||||
|
||||
|
||||
def report_images(report: dict) -> list[str]:
|
||||
return [t["image"] for t in report["targets"] if t["type"] == "image"]
|
||||
|
||||
|
||||
def run(cfg: dict, gitea: GiteaClient, scanner: ScannerClient, now: datetime, emit: Callable[[str], None]) -> dict:
|
||||
"""One triage pass. Any failure raises, and the heartbeat is the last thing it writes."""
|
||||
dry_run = cfg["dry_run"]
|
||||
report = scanner.report()
|
||||
agg = aggregate(report)
|
||||
now_text = now.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
summary: dict = {"dry_run": dry_run, "targets": agg["targets_total"], "critical_with_fix": agg["critical_with_fix"]}
|
||||
summary["dashboard"] = sync_dashboard_issue(gitea, render_dashboard(agg, cfg["page_url"], now), dry_run)
|
||||
|
||||
number = cfg["renovate_issue"]
|
||||
body = _renovate_issue(gitea, number)["body"]
|
||||
remediation = derive_remediation(
|
||||
report_images(report), parse_rows(body), gitea.open_renovate_prs(), parse_detected(body),
|
||||
gitea.pr_url, now_text,
|
||||
)
|
||||
summary["remediation"] = count_states(remediation)
|
||||
|
||||
# Re-read immediately before writing: Renovate rewrites this body, and a tick must apply to
|
||||
# the body as it is now.
|
||||
fresh = _renovate_issue(gitea, number)["body"]
|
||||
ticks = select_ticks(parse_rows(fresh), qualifying_images(report))
|
||||
summary["ticks"] = len(ticks)
|
||||
if ticks and not dry_run:
|
||||
gitea.update_issue(number, {"body": apply_ticks(fresh, ticks)})
|
||||
|
||||
if not dry_run:
|
||||
scanner.triage(now_text, remediation)
|
||||
summary["heartbeat"] = 0 if dry_run else 1
|
||||
summary["fallback_used"] = scanner.used_fallback
|
||||
emit(render_summary(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def render_summary(summary: dict) -> str:
|
||||
verb = "would be" if summary["dry_run"] else "were"
|
||||
dash = summary["dashboard"]
|
||||
states = ", ".join(f"{name} {count}" for name, count in summary["remediation"].items())
|
||||
return "\n".join([
|
||||
f"### CVE triage ({'DRY RUN, no writes' if summary['dry_run'] else 'live'})",
|
||||
"",
|
||||
f"- targets in report: {summary['targets']}",
|
||||
f"- unaccepted CRITICAL with a fix: {summary['critical_with_fix']}",
|
||||
f"- dashboard label created: {dash['label_created']}; issue created: {dash['issue_created']};"
|
||||
f" updated: {dash['issue_updated']}; reopened: {dash['issue_reopened']} ({verb} applied)",
|
||||
f"- remediation states: {states}",
|
||||
f"- approval boxes that {verb} ticked: {summary['ticks']}",
|
||||
f"- heartbeat posted: {summary['heartbeat']}",
|
||||
f"- scanner reached through the ClusterIP fallback: {summary['fallback_used']}",
|
||||
"",
|
||||
])
|
||||
|
||||
|
||||
def load_config(env: dict) -> dict:
|
||||
token = env.get("RENOVATE_TOKEN", "")
|
||||
if not token:
|
||||
raise BotError("RENOVATE_TOKEN is not set")
|
||||
repo = env.get("GITEA_REPOSITORY", "Home-Network/home-network-k3s-impl")
|
||||
if not REPO_RE.fullmatch(repo):
|
||||
raise BotError("config: GITEA_REPOSITORY is malformed")
|
||||
urls = {
|
||||
"gitea_url": env.get("GITEA_URL", "https://gitea.home.keithjackson.dev"),
|
||||
"vuln_url": env.get("VULN_API_URL", "http://vuln-scanner.vuln-scanner.svc.cluster.local:8081"),
|
||||
"page_url": env.get("VULN_PAGE_URL", "https://cve.home.keithjackson.dev"),
|
||||
}
|
||||
fallback = env.get("VULN_API_FALLBACK_URL", "")
|
||||
for name, value in {**urls, **({"fallback": fallback} if fallback else {})}.items():
|
||||
if not HTTP_URL_RE.fullmatch(value):
|
||||
raise BotError(f"config: {name} is not a plain http(s) URL")
|
||||
dry_run = env.get("DRY_RUN", "false")
|
||||
if dry_run not in ("true", "false"):
|
||||
raise BotError("config: DRY_RUN must be true or false")
|
||||
issue = env.get("RENOVATE_DASHBOARD_ISSUE", "91")
|
||||
if not issue.isdigit():
|
||||
raise BotError("config: RENOVATE_DASHBOARD_ISSUE is not a number")
|
||||
return {**urls, "token": token, "repo": repo, "fallback": fallback,
|
||||
"dry_run": dry_run == "true", "renovate_issue": int(issue)}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
def emit(text: str) -> None:
|
||||
print(text)
|
||||
path = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if path:
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(text + "\n")
|
||||
|
||||
try:
|
||||
cfg = load_config(dict(os.environ))
|
||||
transport = UrllibTransport()
|
||||
gitea = GiteaClient(cfg["gitea_url"], cfg["repo"], cfg["token"], transport)
|
||||
scanner = ScannerClient(cfg["vuln_url"], cfg["fallback"], transport)
|
||||
run(cfg, gitea, scanner, datetime.now(timezone.utc), emit)
|
||||
except BotError as exc:
|
||||
print(f"::error::CVE triage failed: {exc}", file=sys.stderr)
|
||||
emit(f"### CVE triage FAILED\n\n{exc}\n")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+373
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# .gitea/workflows/scripts/images.sh — the image pipeline's driver (images.yaml calls this).
|
||||
#
|
||||
# Builds, verifies and (on main) PUBLISHES the three images this repo ships:
|
||||
#
|
||||
# toolbox infra/devcontainer/ the pinned dev/gate toolchain
|
||||
# agent-runner infra/agent/ FROM toolbox; the PM dispatcher + worker
|
||||
# conformance-runner conformance/incluster/ the in-cluster L2/L3 Chainsaw runner
|
||||
#
|
||||
# THE BUILDER (operator decision on PR #81): a ROOTLESS BuildKit daemon, platform/buildkit/,
|
||||
# reached over mTLS. No CI job may reach a privileged daemon, so there is no `docker build` and no
|
||||
# `docker run` here: this script drives buildkitd with a pinned `buildx` (remote driver) and
|
||||
# nothing else. The client certificate arrives as Actions secrets (BUILDKIT_CA,
|
||||
# BUILDKIT_CLIENT_CERT, BUILDKIT_CLIENT_KEY; infra/gitea/buildkit-client-secret.sh puts them
|
||||
# there). If the daemon is unreachable this FAILS LOUDLY — it never skips.
|
||||
#
|
||||
# VERIFICATION IS A BUILD TARGET. Each Containerfile ends in a `verify` stage that runs every
|
||||
# in-image assertion at build time (as the non-root uid, with RUN --network=none); this script
|
||||
# builds `<image>-verify` and only then the image. Build args come from each image's own
|
||||
# `build-image.sh --print-build-args`, the same list the local podman path uses.
|
||||
#
|
||||
# MODES (env MODE):
|
||||
# verify build the verify targets, never push, hold no registry credential. Pull requests.
|
||||
# publish verify, then build + push to ${REGISTRY}/${NAMESPACE}/<image> with tags
|
||||
# `sha-<12 hex>` (immutable: never re-pushed) and `main` (moving; Renovate follows
|
||||
# it). The digest recorded is the one the REGISTRY returns, read back after the push.
|
||||
#
|
||||
# WHAT GETS BUILT (env IMAGES): `auto` (default) diffs BASE_SHA..HEAD_SHA against each image's
|
||||
# inputs; otherwise a space-separated list. A change to the pipeline itself (images.yaml, this
|
||||
# file, platform/buildkit/versions.env) builds everything in verify mode, and nothing extra in
|
||||
# publish mode: re-publishing unchanged inputs would only mint new digests and churn Renovate PRs.
|
||||
#
|
||||
# THE agent-runner BASE:
|
||||
# * verify mode -> FROM the toolbox built in this run, linked with a bake `target:` context —
|
||||
# a pull request holds no registry credential and that toolbox exists in no registry. If the
|
||||
# toolbox's inputs did not change it is built anyway, as the base.
|
||||
# * publish mode, toolbox published in this run -> FROM the digest the registry just returned.
|
||||
# * publish mode otherwise -> AGENT_BASE_IMAGE from infra/agent/versions.env (the published
|
||||
# toolbox digest Renovate keeps current). If that is still the all-zero placeholder the
|
||||
# toolbox has never been published, so it is built and published first (the bootstrap).
|
||||
#
|
||||
# LOOP-FREEDOM. Renovate turns each publish into a digest-bump PR. Those bumps land in
|
||||
# platform/pm-agent/ (no image input), conformance/versions.env (no image input) and
|
||||
# infra/agent/versions.env (agent-runner input -> ONE more agent-runner build, whose own bump
|
||||
# lands in platform/pm-agent/ and stops). The worker Job template is deliberately NOT tracked:
|
||||
# it is baked INTO agent-runner, so bumping it would rebuild agent-runner, mint a new digest,
|
||||
# bump it again, forever. pm-dispatch overwrites that field at runtime anyway.
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
BK_HOST_OVERRIDE="${BUILDKIT_HOST:-}"; BK_NAME_OVERRIDE="${BUILDKIT_SERVERNAME:-}"
|
||||
. ./platform/buildkit/versions.env
|
||||
BUILDKIT_HOST="${BK_HOST_OVERRIDE:-${BUILDKIT_HOST}}"
|
||||
BUILDKIT_SERVERNAME="${BK_NAME_OVERRIDE:-${BUILDKIT_SERVERNAME}}"
|
||||
|
||||
MODE="${MODE:-verify}"
|
||||
IMAGES="${IMAGES:-auto}"
|
||||
REGISTRY="${REGISTRY:-gitea.home.keithjackson.dev}"
|
||||
NAMESPACE="${NAMESPACE:-home-network}"
|
||||
HEAD_SHA="${HEAD_SHA:-$(git rev-parse HEAD)}"
|
||||
BASE_SHA="${BASE_SHA:-}"
|
||||
SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
|
||||
ZERO_DIGEST="sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||
|
||||
ALL_IMAGES=(toolbox agent-runner conformance-runner)
|
||||
SHA_TAG="sha-${HEAD_SHA:0:12}"
|
||||
MOVING_TAG="main"
|
||||
|
||||
log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; printf -- '- **ERROR**: %s\n' "$*" >>"${SUMMARY}"; exit 1; }
|
||||
yn() { if [ -n "$1" ]; then echo "$2"; else echo "$3"; fi; }
|
||||
|
||||
case "${MODE}" in verify|publish) ;; *) die "MODE must be verify or publish, got '${MODE}'" ;; esac
|
||||
for t in git jq curl sha256sum; do command -v "$t" >/dev/null || die "the job image lacks '$t'"; done
|
||||
|
||||
# --- per-image facts ------------------------------------------------------------------------
|
||||
# Inputs: a change under any of these rebuilds that image. conformance-runner takes its pins from
|
||||
# infra/devcontainer/versions.env. conformance/versions.env is NOT an input (it only RECORDS the
|
||||
# published digest), and must not become one: Renovate writes it after every publish -> a loop.
|
||||
inputs() {
|
||||
case "$1" in
|
||||
toolbox) echo "infra/devcontainer/Containerfile infra/devcontainer/versions.env infra/devcontainer/build-image.sh" ;;
|
||||
agent-runner) echo "infra/agent/" ;;
|
||||
conformance-runner) echo "conformance/incluster/Containerfile conformance/incluster/conformance-clone conformance/incluster/build-image.sh infra/devcontainer/versions.env" ;;
|
||||
esac
|
||||
}
|
||||
context_of() { case "$1" in toolbox) echo infra/devcontainer ;; agent-runner) echo infra/agent ;; conformance-runner) echo conformance/incluster ;; esac; }
|
||||
PIPELINE_FILES=".gitea/workflows/images.yaml .gitea/workflows/scripts/images.sh platform/buildkit/versions.env"
|
||||
|
||||
declare -A WANT=() BUILT=() DIGEST=() SECS=() NOTE=()
|
||||
|
||||
changed_any() { # <paths...> -> 0 if BASE_SHA..HEAD_SHA touches any
|
||||
[ -n "$(git diff --name-only "${BASE_SHA}" "${HEAD_SHA}" -- "$@")" ]
|
||||
}
|
||||
|
||||
select_images() {
|
||||
local i
|
||||
if [ "${IMAGES}" != "auto" ]; then
|
||||
for i in ${IMAGES}; do
|
||||
case " ${ALL_IMAGES[*]} " in *" ${i} "*) WANT[$i]=1; NOTE[$i]="requested" ;; *) die "unknown image '${i}'" ;; esac
|
||||
done
|
||||
return
|
||||
fi
|
||||
if [ -z "${BASE_SHA}" ] || [[ "${BASE_SHA}" =~ ^0+$ ]] || ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
|
||||
info "no usable diff base ('${BASE_SHA}'): building everything"
|
||||
for i in "${ALL_IMAGES[@]}"; do WANT[$i]=1; NOTE[$i]="no diff base"; done
|
||||
return
|
||||
fi
|
||||
info "diff base ${BASE_SHA} .. ${HEAD_SHA}"
|
||||
for i in "${ALL_IMAGES[@]}"; do
|
||||
# shellcheck disable=SC2046
|
||||
if changed_any $(inputs "$i"); then WANT[$i]=1; NOTE[$i]="inputs changed"; fi
|
||||
done
|
||||
# shellcheck disable=SC2086
|
||||
if [ "${MODE}" = verify ] && changed_any ${PIPELINE_FILES}; then
|
||||
for i in "${ALL_IMAGES[@]}"; do [ -n "${WANT[$i]:-}" ] || { WANT[$i]=1; NOTE[$i]="pipeline changed"; }; done
|
||||
fi
|
||||
}
|
||||
|
||||
# --- the builder: pinned buildx + mTLS to rootless buildkitd ---------------------------------
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORK}"' EXIT # client key + registry auth live only here, only this long
|
||||
export BUILDX_CONFIG="${WORK}/buildx" DOCKER_CONFIG="${WORK}/docker"
|
||||
# Single-manifest images: the admission policies, Renovate and registry-retention.sh all reason
|
||||
# about one manifest per tag. (Provenance attestations would turn every push into an index.)
|
||||
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
|
||||
mkdir -p "${BUILDX_CONFIG}" "${DOCKER_CONFIG}" "${WORK}/certs"
|
||||
echo '{}' > "${DOCKER_CONFIG}/config.json"
|
||||
|
||||
setup_builder() {
|
||||
log "Builder: rootless buildkitd at ${BUILDKIT_HOST} (mTLS)"
|
||||
BUILDX="${BUILDX_BIN:-}"
|
||||
if [ -z "${BUILDX}" ]; then
|
||||
BUILDX="${WORK}/buildx-bin"
|
||||
curl -fsSL --retry 3 -o "${BUILDX}" \
|
||||
"https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" \
|
||||
|| die "could not download buildx v${BUILDX_VERSION}"
|
||||
echo "${BUILDX_SHA256} ${BUILDX}" | sha256sum -c --quiet - \
|
||||
|| die "buildx v${BUILDX_VERSION} does not match the sha256 pinned in platform/buildkit/versions.env — refusing to run it"
|
||||
chmod +x "${BUILDX}"
|
||||
fi
|
||||
info "client: $("${BUILDX}" version)"
|
||||
|
||||
local missing=()
|
||||
[ -n "${BUILDKIT_CA:-}" ] || missing+=(BUILDKIT_CA)
|
||||
[ -n "${BUILDKIT_CLIENT_CERT:-}" ] || missing+=(BUILDKIT_CLIENT_CERT)
|
||||
[ -n "${BUILDKIT_CLIENT_KEY:-}" ] || missing+=(BUILDKIT_CLIENT_KEY)
|
||||
[ "${#missing[@]}" -eq 0 ] || die "the builder's client certificate is missing: ${missing[*]} not set.
|
||||
These Actions secrets are copied from the cert-manager Secret buildkit/buildkit-client-ci by
|
||||
infra/gitea/buildkit-client-secret.sh (an operator action, after platform/buildkit is
|
||||
deployed). Without them nothing can be built — this fails rather than skipping ON PURPOSE."
|
||||
umask 077
|
||||
printf '%s\n' "${BUILDKIT_CA}" > "${WORK}/certs/ca.pem"
|
||||
printf '%s\n' "${BUILDKIT_CLIENT_CERT}" > "${WORK}/certs/cert.pem"
|
||||
printf '%s\n' "${BUILDKIT_CLIENT_KEY}" > "${WORK}/certs/key.pem"
|
||||
umask 022
|
||||
|
||||
"${BUILDX}" create --name ci-remote --driver remote \
|
||||
--driver-opt "cacert=${WORK}/certs/ca.pem,cert=${WORK}/certs/cert.pem,key=${WORK}/certs/key.pem,servername=${BUILDKIT_SERVERNAME}" \
|
||||
"${BUILDKIT_HOST}" >/dev/null
|
||||
if ! out="$(timeout 90 "${BUILDX}" inspect --bootstrap --builder ci-remote 2>&1)"; then
|
||||
printf '%s\n' "${out}" >&2
|
||||
die "buildkitd is UNREACHABLE at ${BUILDKIT_HOST} — nothing can be built, and this job fails
|
||||
rather than skipping. In order of likelihood:
|
||||
1. platform/buildkit is not deployed yet. Flux applies it only after it merges to main
|
||||
(Kustomization 'buildkit'); a pull request that ADDS it cannot reach it.
|
||||
2. the BUILDKIT_* secrets are stale (the client cert was renewed) or from another CA:
|
||||
re-run infra/gitea/buildkit-client-secret.sh.
|
||||
3. the NetworkPolicy (platform/buildkit/networkpolicy.yaml) does not admit this runner."
|
||||
fi
|
||||
printf '%s\n' "${out}" | grep -E '^(Name|Driver|Status|BuildKit version|Platforms):' | sed 's/^/ /' || true
|
||||
}
|
||||
|
||||
# KEY=VALUE lines from an image's build-image.sh --print-build-args -> a JSON object.
|
||||
build_args_json() {
|
||||
local ctx; ctx="$(context_of "$1")"
|
||||
"./${ctx}/build-image.sh" --print-build-args | jq -Rn '[inputs | select(length>0) | capture("^(?<k>[^=]+)=(?<v>.*)$")] | map({(.k): .v}) | add'
|
||||
}
|
||||
|
||||
# bake_run <bakefile> <targets...>
|
||||
bake_run() {
|
||||
local f="$1"; shift
|
||||
"${BUILDX}" bake --builder ci-remote --progress plain -f "$f" \
|
||||
--metadata-file "${WORK}/meta.json" "$@"
|
||||
}
|
||||
|
||||
# Write a bake file with a verify target and a ship target for one image.
|
||||
# $1 image, $2 agent base mode: "" | "link" | <image ref>
|
||||
write_bake() {
|
||||
local img="$1" base="${2:-}" ctx args f
|
||||
ctx="$(context_of "$img")"
|
||||
args="$(build_args_json "$img")"
|
||||
f="${WORK}/bake-${img}.json"
|
||||
local contexts='{}'
|
||||
if [ "${img}" = agent-runner ]; then
|
||||
if [ "${base}" = link ]; then
|
||||
# FROM ${AGENT_BASE_IMAGE} resolves to the named context below, i.e. the toolbox target
|
||||
# of this same bake file — built here, never pushed anywhere.
|
||||
args="$(jq -c '. + {AGENT_BASE_IMAGE: "toolbox-in-this-run"}' <<<"${args}")"
|
||||
contexts='{"toolbox-in-this-run": "target:toolbox"}'
|
||||
else
|
||||
args="$(jq -c --arg b "${base}" '. + {AGENT_BASE_IMAGE: $b}' <<<"${args}")"
|
||||
fi
|
||||
fi
|
||||
jq -n --arg img "$img" --arg ctx "$ctx" --argjson args "$args" --argjson contexts "$contexts" \
|
||||
--arg tbargs "$( [ "${base}" = link ] && build_args_json toolbox || echo '{}')" \
|
||||
--arg rev "${HEAD_SHA}" --arg repo "${REGISTRY}/${NAMESPACE}/${img}" \
|
||||
--arg sha "${SHA_TAG}" --arg moving "${MOVING_TAG}" --arg mode "${MODE}" '
|
||||
{target: (
|
||||
{
|
||||
($img + "-verify"): {context: $ctx, dockerfile: "Containerfile", target: "verify",
|
||||
args: $args, contexts: $contexts, output: ["type=cacheonly"]},
|
||||
($img): ({context: $ctx, dockerfile: "Containerfile", target: $img, args: $args,
|
||||
contexts: $contexts,
|
||||
labels: {"org.opencontainers.image.revision": $rev}}
|
||||
+ (if $mode == "publish"
|
||||
then {tags: [($repo + ":" + $sha), ($repo + ":" + $moving)], output: ["type=image,push=true"]}
|
||||
else {output: ["type=cacheonly"]} end))
|
||||
}
|
||||
+ (if ($contexts | has("toolbox-in-this-run"))
|
||||
then {toolbox: {context: "infra/devcontainer", dockerfile: "Containerfile", target: "toolbox",
|
||||
args: ($tbargs | fromjson), output: ["type=cacheonly"]}}
|
||||
else {} end)
|
||||
)}' > "$f"
|
||||
echo "$f"
|
||||
}
|
||||
|
||||
# --- registry helpers (publish mode only) ----------------------------------------------------
|
||||
REG_TOKEN=""
|
||||
registry_token() {
|
||||
[ -n "${REG_TOKEN}" ] && return 0
|
||||
REG_TOKEN="$(curl -fsS --config <(printf 'user = "%s:%s"\n' "${REGISTRY_USER}" "${REGISTRY_TOKEN}") \
|
||||
"https://${REGISTRY}/v2/token?service=container_registry&scope=*" | jq -r '.token // empty')" \
|
||||
|| die "could not obtain a registry bearer token from https://${REGISTRY}/v2/token"
|
||||
[ -n "${REG_TOKEN}" ] || die "registry returned no bearer token"
|
||||
}
|
||||
# Prints the manifest digest the REGISTRY serves for <image>:<tag>; nothing if it is absent (404).
|
||||
# Any other answer is fatal: "cannot tell" must never be read as "not published", or the
|
||||
# immutable-tag guard would be bypassed by an auth hiccup.
|
||||
registry_digest() {
|
||||
registry_token
|
||||
local hdrs code digest
|
||||
hdrs="$(curl -sS -I --config <(printf 'header = "Authorization: Bearer %s"\n' "${REG_TOKEN}") \
|
||||
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.oci.image.index.v1+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
"https://${REGISTRY}/v2/${NAMESPACE}/$1/manifests/$2" | tr -d '\r')" \
|
||||
|| die "registry unreachable while reading $1:$2"
|
||||
code="$(awk 'NR==1{print $2}' <<<"${hdrs}")"
|
||||
digest="$(awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}' <<<"${hdrs}" | tail -1)"
|
||||
case "${code}" in
|
||||
200) [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || die "registry answered 200 for $1:$2 with no usable digest"
|
||||
echo "${digest}" ;;
|
||||
404) ;;
|
||||
*) die "registry answered HTTP ${code} for $1:$2 — cannot tell whether it exists" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# build_one <image> [agent base] — verify, then (publish) build + push + read back.
|
||||
build_one() {
|
||||
local img="$1" base="${2:-}" t0 f pushed got existing
|
||||
t0=$(date +%s)
|
||||
f="$(write_bake "$img" "$base")"
|
||||
log "VERIFY ${img} (Containerfile target: verify, on rootless BuildKit)"
|
||||
bake_run "$f" "${img}-verify"
|
||||
BUILT[$img]=1
|
||||
if [ "${MODE}" = publish ]; then
|
||||
existing="$(registry_digest "${img}" "${SHA_TAG}")"
|
||||
# Immutable tag: never overwrite. A rebuild is not byte-reproducible, so re-pushing would
|
||||
# silently change what `sha-<commit>` means after something may already have pinned it.
|
||||
[ -z "${existing}" ] || die "${REGISTRY}/${NAMESPACE}/${img}:${SHA_TAG} already exists (${existing}); tags are immutable"
|
||||
log "BUILD + PUSH ${img} -> ${REGISTRY}/${NAMESPACE}/${img}:{${SHA_TAG},${MOVING_TAG}}"
|
||||
bake_run "$f" "${img}"
|
||||
pushed="$(jq -r --arg t "$img" '.[$t]["containerimage.digest"] // empty' "${WORK}/meta.json")"
|
||||
# The AUTHORITATIVE digest is what the registry serves, read back — not what the client
|
||||
# reported. They should agree; if they do not, nothing downstream may trust either.
|
||||
got="$(registry_digest "${img}" "${SHA_TAG}")"
|
||||
[ -n "${got}" ] || die "pushed ${img}:${SHA_TAG} but the registry does not serve it"
|
||||
[ -z "${pushed}" ] || [ "${pushed}" = "${got}" ] || die "${img}: client reported ${pushed} but the registry serves ${got}"
|
||||
[ "$(registry_digest "${img}" "${MOVING_TAG}")" = "${got}" ] \
|
||||
|| die "${img}: ${MOVING_TAG} does not point at the digest just pushed (${got})"
|
||||
DIGEST[$img]="${got}"
|
||||
info "${img}: registry digest ${got}"
|
||||
fi
|
||||
SECS[$img]=$(( $(date +%s) - t0 ))
|
||||
info "${img}: done in ${SECS[$img]}s"
|
||||
}
|
||||
|
||||
# --- main ---------------------------------------------------------------------------------
|
||||
log "image pipeline: MODE=${MODE} IMAGES=${IMAGES} HEAD=${HEAD_SHA}"
|
||||
select_images
|
||||
|
||||
if [ "${MODE}" = publish ]; then
|
||||
: "${REGISTRY_TOKEN:?REGISTRY_TOKEN is required in publish mode}"
|
||||
REGISTRY_USER="${REGISTRY_USER:-ci-image-publisher}"
|
||||
# buildx (and through it buildkitd, which does the push) reads registry auth from this file.
|
||||
jq -n --arg r "${REGISTRY}" --arg a "$(printf '%s:%s' "${REGISTRY_USER}" "${REGISTRY_TOKEN}" | base64 -w0)" \
|
||||
'{auths: {($r): {auth: $a}}}' > "${DOCKER_CONFIG}/config.json"
|
||||
chmod 600 "${DOCKER_CONFIG}/config.json"
|
||||
registry_token
|
||||
# Publish idempotence: an image already published at THIS commit is not rebuilt. That is what
|
||||
# makes a re-run after a half-finished publish safe — the immutable tag is never pushed twice.
|
||||
for i in "${ALL_IMAGES[@]}"; do
|
||||
[ -n "${WANT[$i]:-}" ] || continue
|
||||
d="$(registry_digest "${i}" "${SHA_TAG}")"
|
||||
if [ -n "${d}" ]; then
|
||||
info "${i}: already published at ${SHA_TAG} (${d}); not rebuilding"
|
||||
unset "WANT[$i]"; DIGEST[$i]="${d}"; NOTE[$i]="already published at this commit"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Decide the agent-runner base (see header).
|
||||
AGENT_BASE=""
|
||||
if [ -n "${WANT[agent-runner]:-}" ]; then
|
||||
PINNED_BASE="$(. ./infra/agent/versions.env; echo "${AGENT_BASE_IMAGE}")"
|
||||
if [ "${MODE}" = verify ]; then
|
||||
AGENT_BASE=link
|
||||
[ -n "${WANT[toolbox]:-}" ] || { WANT[toolbox]=1; NOTE[toolbox]="built as agent-runner's base (verify only)"; }
|
||||
elif [ -z "${WANT[toolbox]:-}" ] && [ -z "${DIGEST[toolbox]:-}" ]; then
|
||||
case "${PINNED_BASE}" in
|
||||
*@"${ZERO_DIGEST}") WANT[toolbox]=1; NOTE[toolbox]="never published; required as agent-runner's base" ;;
|
||||
*@sha256:*) AGENT_BASE="${PINNED_BASE}" ;;
|
||||
*) die "infra/agent/versions.env AGENT_BASE_IMAGE='${PINNED_BASE}' is not digest-pinned; CI will not build on a floating base" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in "${ALL_IMAGES[@]}"; do info "plan: ${i}: $(yn "${WANT[$i]:-}" BUILD skip) (${NOTE[$i]:-inputs unchanged})"; done
|
||||
if [ -z "${WANT[toolbox]:-}${WANT[agent-runner]:-}${WANT[conformance-runner]:-}" ]; then
|
||||
info "nothing to build"
|
||||
else
|
||||
setup_builder
|
||||
fi
|
||||
|
||||
if [ -n "${WANT[toolbox]:-}" ]; then build_one toolbox; fi
|
||||
if [ -n "${WANT[agent-runner]:-}" ]; then
|
||||
if [ -z "${AGENT_BASE}" ]; then
|
||||
AGENT_BASE="${REGISTRY}/${NAMESPACE}/toolbox@${DIGEST[toolbox]:?no toolbox digest to build agent-runner from}"
|
||||
fi
|
||||
info "agent-runner base: $( [ "${AGENT_BASE}" = link ] && echo "the toolbox built in this run (bake target: context)" || echo "${AGENT_BASE}")"
|
||||
build_one agent-runner "${AGENT_BASE}"
|
||||
fi
|
||||
if [ -n "${WANT[conformance-runner]:-}" ]; then build_one conformance-runner; fi
|
||||
|
||||
# --- summary ------------------------------------------------------------------------------
|
||||
{
|
||||
echo "## Images (${MODE})"
|
||||
echo
|
||||
echo "commit \`${HEAD_SHA}\` — builder: rootless BuildKit \`${BUILDKIT_HOST}\` (mTLS)"
|
||||
echo
|
||||
echo "| image | verify target built | seconds | published digest (registry-returned) | note |"
|
||||
echo "|---|---|---|---|---|"
|
||||
for i in "${ALL_IMAGES[@]}"; do
|
||||
d="${DIGEST[$i]:-}"
|
||||
echo "| \`${i}\` | $(yn "${BUILT[$i]:-}" yes no) | ${SECS[$i]:--} | $(yn "${d}" "\`${REGISTRY}/${NAMESPACE}/${i}@${d}\`" -) | ${NOTE[$i]:-inputs unchanged} |"
|
||||
done
|
||||
echo
|
||||
if [ "${MODE}" = publish ]; then
|
||||
echo "Tags pushed: \`${SHA_TAG}\` (immutable) and \`${MOVING_TAG}\` (moving)."
|
||||
echo "Nothing here writes to git. Renovate opens the digest-bump PR on its next run."
|
||||
else
|
||||
echo "Verify mode: nothing was pushed and no registry credential was present."
|
||||
fi
|
||||
} >>"${SUMMARY}"
|
||||
log "done"
|
||||
@@ -37,6 +37,14 @@ coverage/
|
||||
infra/tofu/backend.hcl
|
||||
infra/tofu/.r2.env
|
||||
|
||||
# Operator scratch, writable from inside the dev container (the repo is its only persistent
|
||||
# read-write mount). Holds things that must never be committed: a staged copy of the SOPS age or
|
||||
# WireGuard key, rendered VPN client profiles (infra/vpn-ingress/new-peer.sh writes here by
|
||||
# default). `infra/ansible/.gitignore` already does this for its own .cache/; this is the same
|
||||
# convention at the root. Do NOT rely on `*.key` alone to protect a staged key — that only covers
|
||||
# one filename shape, which is why this rule exists.
|
||||
.cache/
|
||||
|
||||
# Claude Code local scratch: agent worktrees + session state. Nothing here is tracked;
|
||||
# ignoring it keeps `git add -A` from staging embedded worktree repos and keeps the lint
|
||||
# stage (which scans untracked files) from tripping on .claude/worktrees/.
|
||||
|
||||
@@ -4,6 +4,125 @@ Read this before doing anything durable. It is the delivery contract for AI codi
|
||||
sessions and agents. The newest entry in `decisions/` is the current PM handoff and
|
||||
takes precedence on *what* to work on; this file governs *how* it lands.
|
||||
|
||||
**Start with §0.** It is first because it is the rule you can break before you have read
|
||||
anything else: the very first `git checkout -b` in the wrong directory has already done the
|
||||
damage.
|
||||
|
||||
## 0. Work in your OWN workspace. Never in the shared checkout.
|
||||
|
||||
Before you create a branch, edit a file, or run `git` anything: **make yourself a git
|
||||
worktree outside the repository root, and do all of your work there.** One workspace per
|
||||
agent run, no exceptions.
|
||||
|
||||
# from the shared checkout, ONCE, at the start of your run
|
||||
git fetch origin main
|
||||
git worktree add -b <type>/<slug> /path/outside/the/repo/wt-<slug> origin/main
|
||||
cd /path/outside/the/repo/wt-<slug> # everything after this happens HERE
|
||||
|
||||
`/path/outside/the/repo` means outside this repository's tree — your session scratchpad,
|
||||
or a sibling like `/var/home/keith/source/wt-<slug>`. Branch from **`origin/main`**, never
|
||||
from `main` and never from `HEAD`; see "stale base" below.
|
||||
|
||||
**The shared checkout is `/var/home/keith/source/home-network-k3s-impl`** — the operator's
|
||||
working copy, the one you are probably started in. In it, do **not**:
|
||||
|
||||
- **`git checkout` / `git switch` / `git checkout -b`.** Its HEAD is shared state; moving
|
||||
it moves the tree out from under whoever else is using it.
|
||||
- **edit, create or delete a file.** Including "just one doc", including a file you intend
|
||||
to move later.
|
||||
- **`git add` / `git commit` / `git stash` / `git restore`.** `git add -A` there stages
|
||||
whatever another agent left lying around; `git stash` *removes* their uncommitted work
|
||||
into a place they will not look.
|
||||
- **`git worktree remove` anything you did not create.**
|
||||
|
||||
Read-only commands there are fine, and the one write you do need is creating your own
|
||||
worktree.
|
||||
|
||||
### Why — measured in the shared checkout, 2026-10-05
|
||||
|
||||
| What was measured | Value |
|
||||
|---|---|
|
||||
| `checkout: moving` entries in its reflog | **107** — the tree has been yanked between branches 107 times |
|
||||
| the branch it was sitting on | `feat/pm-agent-issue-to-pr` — **not `main`** |
|
||||
| uncommitted trees left in it by other runs | **2** (`infra/agent/`, `platform/pm-agent/`), mtimes five minutes apart the same evening |
|
||||
| its local `main` vs `origin/main` | **two commits behind** (`4386d14` vs `22c94ef`) |
|
||||
| registered worktrees | **42**, of which **31** are on branches already merged to `main` |
|
||||
| gate scripts carrying an explicit `.claude/` exclusion | **9** |
|
||||
|
||||
Each row is a distinct way the shared checkout damages work:
|
||||
|
||||
- **Dirty tree, foreign files.** Two other runs' uncommitted directories were sitting in
|
||||
it. An agent that edits there and runs `git add -A` ships someone else's half-finished
|
||||
work inside its own PR, and the PR's author has no idea what those files are. This is not
|
||||
hypothetical and it is not historical: **eleven minutes after those two directories were
|
||||
measured, while this very section was being written, another run committed them** —
|
||||
`5334abe feat(pm-agent): ...`, 1,000+ lines across `infra/agent/`, `clusters/`, `catalog/`
|
||||
and `apps/` — onto the shared checkout's checked-out branch, moving its HEAD under anyone
|
||||
reading it. Had this rule's author been working in that tree rather than in a worktree,
|
||||
that commit and this one would have collided.
|
||||
- **Stale base.** Its `main` was two commits behind `origin/main`, so
|
||||
`git checkout -b x main` there branches off a tree that is already not what CI will merge
|
||||
into — the gate you run locally proves a base nobody else has. Worse, it was not even *on*
|
||||
`main`: a bare `git checkout -b x` would have branched off `feat/pm-agent-issue-to-pr` and
|
||||
silently inherited that work into your PR. `git fetch` first, and name `origin/main`
|
||||
explicitly as the start point.
|
||||
- **Shared HEAD.** 107 branch switches in one tree is 107 chances to switch it while
|
||||
another agent is mid-edit. The loser's changes are either carried onto the wrong branch
|
||||
or lost to a checkout conflict — and the symptom surfaces in *their* session, not yours.
|
||||
|
||||
### Put the workspace OUTSIDE the repo root
|
||||
|
||||
A worktree *inside* the tree (`.claude/worktrees/agent-*/` — which is where Claude Code's
|
||||
own default puts it) is a full second copy of the repo that every tree-walking check then
|
||||
finds. Measured 2026-09-12: `find`-based discovery reported **the same manifest under test
|
||||
once per worktree, up to 18 times**, and the L0 check comparing that listing against the
|
||||
tree failed on paths that are not authored surface at all
|
||||
(`platform/monitoring/tests/run.sh:115-123`). Nine gate scripts now carry an explicit
|
||||
`.claude/` prune for this reason — which means the hazard is *worked around*, not fixed,
|
||||
and the next check that walks the tree will hit it again. Keep the copy out of the tree and
|
||||
none of that applies.
|
||||
|
||||
### Clean up — and expect not to be able to
|
||||
|
||||
31 of the 42 registered worktrees are on branches already merged to `main`. A registered
|
||||
worktree **pins its branch**: while it exists, that branch cannot be deleted or checked out
|
||||
anywhere else. So a left-behind workspace is a lock on a branch name, and the cost lands on
|
||||
whoever next needs it.
|
||||
|
||||
Remove yours when you are done:
|
||||
|
||||
git worktree remove /path/outside/the/repo/wt-<slug>
|
||||
|
||||
Measured 2026-10-05: on a worktree **you created yourself, in the session scratchpad**, that
|
||||
command **succeeds** — rc 0, no prompt, with or without a branch checked out. The "worktree
|
||||
removal is sandbox-blocked" note in circulation is about the `.claude/worktrees/agent-*` ones
|
||||
belonging to *other* sessions, some of which are additionally `locked` on git's own terms. So:
|
||||
remove yours, and only yours. If removal is refused, say so in your handoff and leave the
|
||||
worktree registered — do **not** `rm -rf` the directory, which leaves a registered worktree
|
||||
pointing at nothing and still holding the branch.
|
||||
|
||||
If someone *else's* stale worktree is holding the branch you need, do not try to reclaim the
|
||||
branch name. Push a `rework/<pr-number>` branch to the PR's refspec instead.
|
||||
|
||||
### If you are already standing in the shared checkout with edits
|
||||
|
||||
Stop. Do not `git stash` (that moves other agents' work too). Create your worktree as
|
||||
above, redo the edits there, and leave the shared checkout's HEAD and working tree exactly
|
||||
as you found them.
|
||||
|
||||
### Gating from a worktree
|
||||
|
||||
Three known breakages, all with workarounds, all required every run: see §3, "Running the
|
||||
gate from a worktree".
|
||||
|
||||
### If you are writing a brief for another agent
|
||||
|
||||
The brief must name **its own explicit branch** and tell it to create **its own worktree**
|
||||
from `origin/main`. "Work on the current branch" and "continue in the checkout" are how
|
||||
branch sprawl and cross-contamination got here: an agent given no workspace uses the one it
|
||||
is started in, which is the shared one. Also carry §3's three gate workarounds into the
|
||||
brief — an agent that cannot gate will report rc 3 as green.
|
||||
|
||||
## 1. Never push to `main`. Ever.
|
||||
|
||||
Every durable change reaches `main` exactly one way:
|
||||
@@ -147,15 +266,21 @@ Exit codes are a contract — do not flatten them:
|
||||
| 3 | UNDETERMINED — a stage did not run. **Never report this as green.** |
|
||||
|
||||
`spec-immutability` reaches rc 3 whenever the sibling spec repo is not checked out
|
||||
beside this one; that single skip is expected and tolerated. Any *other* skip is a
|
||||
real gap in your run.
|
||||
beside this one; that single skip is expected and tolerated — and from a worktree it is
|
||||
avoidable outright (§3's worktree list, item 3). `gitea-settings` is the other expected
|
||||
skip: it needs `SETTINGS_API_TOKEN`, which exists only in CI, so **a local run cannot
|
||||
return 0** — the honest best local result is rc 3 with `gitea-settings` as the only stage
|
||||
that did not run. Any *further* skip is a real gap in your run. Report the rc and name the
|
||||
skipped stages; never round rc 3 up.
|
||||
|
||||
**Never gate on the host.** `promtool` and `kustomize` are not on the host `$PATH`,
|
||||
so checks SKIP and you get a false rc 3. Only the pinned toolbox result counts.
|
||||
|
||||
### Running the gate from a git worktree
|
||||
### Running the gate from a worktree
|
||||
|
||||
Two unrelated failures, both with known workarounds — put both in any agent brief:
|
||||
This is the **normal** path, not an edge case — §0 requires you to be in a worktree, so
|
||||
every gate run you do hits all of these. Three known breakages, all with
|
||||
workarounds, all of them yours to apply every run — put them in any agent brief:
|
||||
|
||||
1. **`dev.sh` refuses to start:** `infra/ansible/.cache/prod.kubeconfig` is untracked
|
||||
and exists only in the main checkout. Do **not** copy it (it is a credential).
|
||||
@@ -166,10 +291,17 @@ Two unrelated failures, both with known workarounds — put both in any agent br
|
||||
the `ENGINE` hook — a shim that intercepts `run` and re-execs
|
||||
`podman run -v /path/to/main/.git:/path/to/main/.git:ro "$@"`. A traceback in
|
||||
`tracked_files()` is this bug, not a broken tree.
|
||||
3. **`spec-immutability` skips** — the skip called tolerated above is also *avoidable*. `dev.sh`
|
||||
derives the sibling spec path from its own `REPO_ROOT`, which in a worktree is your
|
||||
workspace, so `../home-network-k3s` does not resolve and it adds no mount. Mount the spec
|
||||
repo read-only in the same `ENGINE` shim and pass `-e SPEC_REPO=/home-network-k3s`: with
|
||||
both mounts the stage reports `pass`, not SKIPPED, and R-TEST-06 is actually covered
|
||||
(measured — `decisions/2026-10-01-render-selftest-runs-in-gate.md`).
|
||||
|
||||
## 4. Branch naming
|
||||
|
||||
Use `feat/`, `fix/`, `docs/`, `chore/`. Note that `.gitea/workflows/gate.yml`'s **push**
|
||||
Use `feat/`, `fix/`, `docs/`, `chore/`. **One branch, one workspace, one PR** — if you
|
||||
need a second branch, make a second worktree (§0); do not re-point an existing one. Note that `.gitea/workflows/gate.yml`'s **push**
|
||||
trigger lists `feature/**` — not `feat/**` — so branch pushes get no fast-feedback run.
|
||||
The `pull_request` trigger fires regardless of branch name, so the PR gate is unaffected.
|
||||
|
||||
|
||||
@@ -11,6 +11,9 @@ specification** at [`../home-network-k3s`](../home-network-k3s).
|
||||
- **Done = a green, reproducible, requirement-tagged conformance test** — never "merged," never a
|
||||
manual demo (spec `docs/14`, `agents/definition-of-done.md`).
|
||||
- **No work on `main`.** Every change is a feature branch merged into `main` (spec `CONTRIBUTING.md`).
|
||||
- **No work in a shared checkout.** Every agent run gets its own git worktree, outside the repo
|
||||
root, branched from `origin/main`. This checkout is shared mutable state — see
|
||||
[`CLAUDE.md` §0](CLAUDE.md).
|
||||
- **No plaintext secrets** — SOPS-encrypted only (spec `docs/07`).
|
||||
- **Phase discipline** — only build what the active roadmap phase authorizes (spec `roadmap/phases.md`).
|
||||
|
||||
@@ -21,7 +24,8 @@ infra/ node provisioning: OpenTofu (Proxmox VE) + cloud-init + Ansible,
|
||||
bootstrap/ one-time cluster bootstrap: Flux install, SOPS/age key, host Gitea
|
||||
platform/ platform CONTROLLERS (Flux HelmReleases / operator bases): kyverno, cilium,
|
||||
longhorn, external-snapshotter, cert-manager, externaldns, gateway, coredns,
|
||||
vpn-egress, monitoring, cnpg, keycloak, oauth2-proxy, kubevirt
|
||||
vpn-egress, vpn-ingress, monitoring, cnpg, keycloak, oauth2-proxy, kubevirt
|
||||
(vpn-egress = pods OUT via PIA; vpn-ingress = remote clients IN via WireGuard)
|
||||
platform-config/ platform CONFIGS — the custom resources of the CRDs installed above
|
||||
(ClusterIssuers, Gateway, Kyverno policies, KubeVirt/CDI CRs, Keycloak CRs).
|
||||
Split out because CRs and the CRDs they use cannot apply in one pass.
|
||||
@@ -72,7 +76,7 @@ bug but the honest answer, because proofs are claims about a specific tree on a
|
||||
|---|---|
|
||||
| 0 Foundations | ServiceDefinition schema + renderer, Kyverno L0 gate, conformance harness, CI |
|
||||
| 1 Cluster bring-up | 3-node k3s HA, embedded etcd, kube-vip VIP, Cilium, Flux, SOPS. Originally Proxmox VMs; **now 3× mini-PC bare metal** (`k3s-prod-1/2/3`, `192.168.2.10/.11/.12`, k3s v1.36.4+k3s1, Debian 13) |
|
||||
| 2 Platform core | Cilium · Longhorn · cert-manager (LE-prod wildcard) · ExternalDNS · Gateway API · split-horizon DNS, **both halves**: CoreDNS for pods, and **PiHole as the authoritative LAN resolver** for `${BASE_DOMAIN}` on udp/53 + tcp/53 at a pinned VIP (`192.168.2.23` on prod). The LAN half was claimed from the start but did not exist until 2026-09-14 — PiHole's Service was ClusterIP :80/:4180, so LAN clients ran on hand-maintained per-client `dnsmasq` files |
|
||||
| 2 Platform core | Cilium · Longhorn · cert-manager (LE-prod wildcard) · ExternalDNS · Gateway API · split-horizon DNS, **both halves**: CoreDNS for pods, and **PiHole as the LAN resolver**, answering an explicit list of `${BASE_DOMAIN}` names (LAN-only apps, gitea/nextcloud/keycloak/auth, the L4 VIPs — everything else resolves publicly to Cloudflare, since 2026-11-09) on udp/53 + tcp/53 at a pinned VIP (`192.168.2.23` on prod). The LAN half was claimed from the start but did not exist until 2026-09-14 — PiHole's Service was ClusterIP :80/:4180, so LAN clients ran on hand-maintained per-client `dnsmasq` files |
|
||||
| 3 Egress / VPN | ADR-0015 node-level OpenVPN split-tunnel + Cilium egress-gateway steering + host kill-switch |
|
||||
| 4 Observability | kube-prometheus-stack · Loki · Alloy · Alertmanager. **Velero unblocked 2026-09-13** — backs up to Cloudflare R2 (OQ-BKP closed; the `velero-plugin-for-aws` tagging incompatibility needed a pinned plugin). Longhorn has **no** backup target by decision: volume data is protected by Velero + kopia to R2. image-automation authored but disabled |
|
||||
| 5 IaaS | KubeVirt + CDI on real nested KVM; live-migratable RWX VM storage. The UniFi VM runs, but **UniFi OS Server is not yet installed in the guest** |
|
||||
@@ -94,6 +98,20 @@ The running narrative — decisions, root causes, incidents, follow-ups — is
|
||||
|
||||
## Working on this repo
|
||||
|
||||
**First, get out of this checkout.** It is the operator's working copy and several agents
|
||||
share it; concurrent runs here have overwritten each other's branches and left each other's
|
||||
uncommitted files behind. One workspace per change:
|
||||
|
||||
```bash
|
||||
git fetch origin main
|
||||
git worktree add -b <type>/<slug> /path/outside/the/repo/wt-<slug> origin/main
|
||||
cd /path/outside/the/repo/wt-<slug>
|
||||
```
|
||||
|
||||
Outside the repo root specifically — a worktree under `.claude/worktrees/` is a second full
|
||||
copy that tree-walking gate checks then count twice. The rule, the measurements behind it, and
|
||||
the three gate workarounds a worktree needs are in [`CLAUDE.md` §0](CLAUDE.md) and §3.
|
||||
|
||||
Everything the gate needs is baked into the pinned dev container — **do not install the
|
||||
toolchain on the host**:
|
||||
|
||||
|
||||
@@ -100,6 +100,11 @@ metadata:
|
||||
# external-dns (source: service) publishes A unifi.home.keithjackson.dev -> the pinned VIP.
|
||||
# This is the browser + device target; UI is https://unifi.home.keithjackson.dev:11443.
|
||||
external-dns.alpha.kubernetes.io/hostname: "unifi.${BASE_DOMAIN}"
|
||||
# DNS-ONLY, NEVER PROXIED (2026-11-08). external-dns runs with --cloudflare-proxied globally (the
|
||||
# gateway names point at the Cloudflare Tunnel); Cloudflare's proxy carries HTTP(S) only, so an
|
||||
# orange-clouded A for these L4 ports would be broken. Required by the L0 check
|
||||
# internal-services-publish-no-public-dns (9b) on every `service`-source publisher.
|
||||
external-dns.alpha.kubernetes.io/cloudflare-proxied: "false"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Cluster
|
||||
|
||||
@@ -176,8 +176,8 @@ or any UDP (`3478`/`10001`) — exactly the ports adoption rides. So:
|
||||
|
||||
**UI access:** `https://unifi.home.keithjackson.dev:11443`. That name resolves to the VIP and not to
|
||||
the Gateway on purpose: both `platform/coredns/coredns-custom.yaml` (in-cluster) and
|
||||
`apps/wave-0/pihole.yaml` (LAN split-horizon) carve `unifi.${BASE_DOMAIN}` out of the
|
||||
zone-wide `${GATEWAY_VIP}` answer and point it at `${UNIFI_L4_VIP}`. `${BASE_DOMAIN}` on prod is the
|
||||
`apps/wave-0/pihole.yaml` (LAN split-horizon, an explicit name list since 2026-11-09) point
|
||||
`unifi.${BASE_DOMAIN}` at `${UNIFI_L4_VIP}` rather than `${GATEWAY_VIP}`. `${BASE_DOMAIN}` on prod is the
|
||||
apex `home.keithjackson.dev` (confirmed live in the `cluster-vars` ConfigMap) after the 2026-09-13
|
||||
flip, so this hostname is current, not a dev leftover.
|
||||
|
||||
|
||||
+28
-3
@@ -78,7 +78,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: gitea
|
||||
image: gitea/gitea:1.24.3
|
||||
image: gitea/gitea:28.3.0
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
@@ -89,14 +89,18 @@ spec:
|
||||
secretKeyRef:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED
|
||||
value: 'true'
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL
|
||||
value: github
|
||||
- name: GITEA__actions__RUN_RETENTION_DAYS
|
||||
value: '400'
|
||||
- name: GITEA__security__EGRESS_MODE
|
||||
value: lax
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: 'true'
|
||||
- name: GITEA_ADMIN_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -191,6 +195,27 @@ spec:
|
||||
- port: '3000'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy in-cluster client ingress allow (R-NET-02): gitea-allow-pm-agent-ingress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: gitea-allow-pm-agent-ingress
|
||||
namespace: gitea
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: pm-agent
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '3000'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): gitea-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
|
||||
@@ -78,14 +78,14 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: jackett
|
||||
image: lscr.io/linuxserver/jackett:latest
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3030-ls30
|
||||
ports:
|
||||
- containerPort: 9117
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
# Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered
|
||||
# nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
@@ -300,7 +300,7 @@ spec:
|
||||
# is already in it and it is already present on every node, so this
|
||||
# adds no new image and no new supply-chain surface. Keep the tag in
|
||||
# step with spec.imageName above when Renovate bumps it.
|
||||
image: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
image: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- psql
|
||||
|
||||
@@ -83,15 +83,25 @@
|
||||
# and would also fix the same latent defect in unifi and crafty; it belongs in its own change.
|
||||
#
|
||||
# -------------------------------------------------------------------------------------------------
|
||||
# loadBalancerSourceRanges — THE LAN RESTRICTION, ENFORCED IN THE DATAPATH.
|
||||
# loadBalancerSourceRanges — THE LAN + VPN RESTRICTION, ENFORCED IN THE DATAPATH.
|
||||
# -------------------------------------------------------------------------------------------------
|
||||
# Cilium enforces this list in eBPF on the ingress node, on the VIP, BEFORE load-balancing and
|
||||
# before SNAT. A packet to ${PIHOLE_DNS_VIP}:53 from outside ${LAN_CIDR} is dropped by the
|
||||
# datapath — it never reaches a backend and never reaches a network policy. This is what makes
|
||||
# before SNAT. A packet to ${PIHOLE_DNS_VIP}:53 from outside ${LAN_CIDR} and ${WG_CLIENT_CIDR} is
|
||||
# dropped by the datapath — it never reaches a backend and never reaches a network policy. This is what makes
|
||||
# "LAN clients only" a real, tested restriction rather than a comment: VERIFIED by narrowing the
|
||||
# range to a /24 that excludes the test client, after which the same query that had been answering
|
||||
# stopped being answered at all, and restoring it, after which it answered again.
|
||||
#
|
||||
# ${WG_CLIENT_CIDR} IS LISTED BECAUSE VPN CLIENTS REACH THIS CHECK UNTRANSLATED. wg0 is one of
|
||||
# Cilium's kube-proxy-replacement devices (measured 2026-11-08: `cilium-dbg status` lists eno1, wg0,
|
||||
# tun0), so a tunnel client's query is load-balanced as it enters from wg0, ahead of the iptables
|
||||
# MASQUERADE in platform/vpn-ingress/daemonset.yaml, and the source tested here is still 10.90.0.x
|
||||
# rather than the node's address. Without this entry a full-tunnel client's DNS is predicted to die
|
||||
# with DROP_NOT_IN_SRC_RANGE (derived from Cilium's source; NOT observed live, because no peer has
|
||||
# ever completed a handshake). The range is private and reachable only by a key in peers.conf, so
|
||||
# PiHole stays off the internet; conformance/tests/l0-static/pihole-dns-source-ranges fails the gate
|
||||
# if either entry goes missing or any range here stops being RFC1918.
|
||||
#
|
||||
# It is also the layer that survives the SNAT above. A CiliumNetworkPolicy cannot express this
|
||||
# restriction once the source has been rewritten: measured on this cluster, the forwarded query
|
||||
# arrives from the ingress node's internal router address and Cilium resolves that source to the
|
||||
@@ -113,6 +123,7 @@ spec:
|
||||
externalTrafficPolicy: Cluster
|
||||
loadBalancerSourceRanges:
|
||||
- ${LAN_CIDR}
|
||||
- ${WG_CLIENT_CIDR}
|
||||
selector:
|
||||
app.kubernetes.io/name: pihole
|
||||
ports:
|
||||
@@ -133,6 +144,7 @@ spec:
|
||||
externalTrafficPolicy: Cluster
|
||||
loadBalancerSourceRanges:
|
||||
- ${LAN_CIDR}
|
||||
- ${WG_CLIENT_CIDR}
|
||||
selector:
|
||||
app.kubernetes.io/name: pihole
|
||||
ports:
|
||||
@@ -158,7 +170,9 @@ spec:
|
||||
# "Policy denied ... identity world" drops in `cilium-dbg monitor`). So `world` is the only peer
|
||||
# selector that can describe LAN traffic that has been through the LoadBalancer, and the set of
|
||||
# addresses that can actually BE that traffic is fixed by loadBalancerSourceRanges before this
|
||||
# policy is consulted. Together: ${LAN_CIDR} in the datapath, udp/53 + tcp/53 here.
|
||||
# policy is consulted. Together: ${LAN_CIDR} + ${WG_CLIENT_CIDR} in the datapath, udp/53 + tcp/53
|
||||
# here. A VPN client delivered to a backend on its own ingress node keeps its 10.90.0.x source,
|
||||
# which Cilium also classifies as `world`, so this rule needs no new peer for it.
|
||||
#
|
||||
# This is NOT wider than the precedent it follows. unifi-allow-l4-ingress and crafty-allow-ingress
|
||||
# admit `fromEntities: [world, cluster, host]` on their L4 ports with NO source-range filter at all;
|
||||
@@ -187,7 +201,7 @@ spec:
|
||||
app.kubernetes.io/name: pihole
|
||||
ingress:
|
||||
- fromEntities:
|
||||
- world # LAN clients, post-LoadBalancer: see the block above for why not fromCIDR
|
||||
- world # LAN + VPN clients, post-LoadBalancer: see the block above for why not fromCIDR
|
||||
- host # this node's kubelet probes
|
||||
- remote-node # the node-to-node hop that forwards a query to wherever the pod runs
|
||||
toPorts:
|
||||
|
||||
@@ -62,7 +62,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: pihole
|
||||
image: pihole/pihole
|
||||
image: pihole/pihole:2026.10.1
|
||||
ports:
|
||||
- containerPort: 80
|
||||
env:
|
||||
@@ -73,7 +73,7 @@ spec:
|
||||
- name: FTLCONF_dns_rateLimit_count
|
||||
value: '0'
|
||||
- name: FTLCONF_misc_dnsmasq_lines
|
||||
value: address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}
|
||||
value: local-ttl=60;address=/gitea.${BASE_DOMAIN}/${GATEWAY_VIP};address=/nextcloud.${BASE_DOMAIN}/${GATEWAY_VIP};address=/keycloak.${BASE_DOMAIN}/${GATEWAY_VIP};address=/auth.${BASE_DOMAIN}/${GATEWAY_VIP};address=/headlamp.${BASE_DOMAIN}/${GATEWAY_VIP};address=/cve.${BASE_DOMAIN}/${GATEWAY_VIP};address=/jackett.${BASE_DOMAIN}/${GATEWAY_VIP};address=/pihole.${BASE_DOMAIN}/${GATEWAY_VIP};address=/plex.${BASE_DOMAIN}/${GATEWAY_VIP};address=/qbittorrent.${BASE_DOMAIN}/${GATEWAY_VIP};address=/radarr.${BASE_DOMAIN}/${GATEWAY_VIP};address=/semaphore.${BASE_DOMAIN}/${GATEWAY_VIP};address=/sonarr.${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}
|
||||
- name: FTLCONF_webserver_api_password
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -90,7 +90,7 @@ spec:
|
||||
- -c
|
||||
- "# Assert the declared adlists in gravity.db. Idempotent: steady state writes nothing and\n# does NOT run a gravity rebuild.\nset -u\nGRAVITY=/etc/pihole/gravity.db\n\n# THE DECLARED SET \u2014 one URL per line, the source of truth for what PiHole blocks.\n# To add a list: add a line here, commit, let Flux reconcile. Nothing else to do.\nDECLARED='https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'\n\n# On a fresh PVC, FTL creates gravity.db a moment after the container starts. Wait, bounded,\n# and NEVER block or fail the start: a missing DB here must not crashloop DNS for the LAN.\ni=0\nwhile [ ! -s \"$GRAVITY\" ] && [ \"$i\" -lt 90 ]; do sleep 2; i=$(expr $i + 1); done\nif [ ! -s \"$GRAVITY\" ]; then\n echo \"pihole-adlists: gravity.db absent after 180s - leaving adlists untouched\" >&2\n exit 0\nfi\n\nsql() { pihole-FTL sqlite3 \"$GRAVITY\" \"$1\"; }\nCHANGED=0\n\n# 1) Every declared URL must be present AND enabled.\necho \"$DECLARED\" | while IFS= read -r url; do\n [ -n \"$url\" ] || continue\n printf '%s\\n' \"$url\"\ndone > /tmp/declared.txt\nwhile IFS= read -r url; do\n [ -n \"$url\" ] || continue\n # Single quotes inside a URL would break this SQL; adlist URLs do not contain them.\n have=$(sql \"select count(*) from adlist where address='$url';\")\n if [ \"$have\" = \"0\" ]; then\n echo \"pihole-adlists: adding $url\"\n sql \"insert into adlist (address,enabled,comment) values ('$url',1,'declared in catalog/pihole.yaml');\"\n CHANGED=1\n else\n en=$(sql \"select enabled from adlist where address='$url';\")\n if [ \"$en\" != \"1\" ]; then\n echo \"pihole-adlists: re-enabling $url\"\n sql \"update adlist set enabled=1 where address='$url';\"\n CHANGED=1\n fi\n fi\ndone < /tmp/declared.txt\n\n# 2) Anything enabled in the DB but NOT declared gets disabled (never deleted - see above).\nsql \"select address from adlist where enabled=1;\" | while IFS= read -r addr; do\n [ -n \"$addr\" ] || continue\n if ! grep -qxF \"$addr\" /tmp/declared.txt; then\n echo \"pihole-adlists: disabling undeclared $addr\"\n sql \"update adlist set enabled=0 where address='$addr';\"\n echo changed >> /tmp/adlist-changed\n fi\ndone\n[ -s /tmp/adlist-changed ] && CHANGED=1\nrm -f /tmp/adlist-changed /tmp/declared.txt\n\n# 3) Rebuild gravity ONLY if the declared set actually moved.\nif [ \"$CHANGED\" = \"1\" ]; then\n echo \"pihole-adlists: declared set changed - running gravity update\"\n pihole -g || echo \"pihole-adlists: gravity update failed; existing gravity.db retained\" >&2\nelse\n echo \"pihole-adlists: already matches the declared set - nothing to do\"\nfi\nexit 0\n"
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
storage: 16Gi
|
||||
---
|
||||
# PVC plex-media (R-STATE)
|
||||
apiVersion: v1
|
||||
@@ -120,7 +120,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: plex
|
||||
image: plexinc/pms-docker:latest
|
||||
image: plexinc/pms-docker:1.43.6.11085-d1e52e169
|
||||
ports:
|
||||
- containerPort: 32400
|
||||
env:
|
||||
@@ -302,6 +302,8 @@ metadata:
|
||||
namespace: plex
|
||||
labels:
|
||||
app.kubernetes.io/name: plex
|
||||
annotations:
|
||||
external-dns.alpha.kubernetes.io/controller: none
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: qbittorrent
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2
|
||||
image: lscr.io/linuxserver/qbittorrent:5.3.0
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
@@ -156,7 +156,7 @@ spec:
|
||||
, \"[Preferences]\"\n\ndef matches(raw):\n \"\"\"True iff the stored digest is one OUR password produces (salt is read from it).\"\"\"\n v = raw.strip().strip('\"')\n if v.startswith(\"@ByteArray(\") and v.endswith(\")\"):\n v = v[len(\"@ByteArray(\"):-1]\n try:\n salt_b64, digest_b64 = v.split(\":\", 1)\n salt = base64.b64decode(salt_b64, validate=True)\n digest = base64.b64decode(digest_b64, validate=True)\n except Exception:\n return False\n if not salt or not digest:\n return False\n return hmac.compare_digest(\n hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, len(digest)), digest)\n\ntry:\n lines = open(conf, encoding=\"utf-8\", errors=\"replace\").read().splitlines()\nexcept FileNotFoundError:\n lines = []\n\ndef value_of(key):\n for i, line in enumerate(lines):\n if line.startswith(key + \"=\"):\n return i, line.split(\"=\", 1)[1]\n return -1, None\n\nui, uval = value_of(UKEY)\npi, pval = value_of(PKEY)\nif uval == user and pval is not None and matches(pval):\n raise SystemExit(0) # converged - leave the file (and its salt) alone\n\nsalt = os.urandom(SALT_LEN)\ndigest = hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, DIGEST_LEN)\nstored = base64.b64encode(salt).decode() + \":\" + base64.b64encode(digest).decode()\ndesired = [UKEY + \"=\" + user, PKEY + '=\"@ByteArray(' + stored + ')\"']\n\nout = [line for i, line in enumerate(lines) if i not in (ui, pi)]\nif SECTION in out:\n at = out.index(SECTION) + 1\nelse:\n if out and out[-1].strip():\n out.append(\"\")\n out.append(SECTION)\n at = len(out)\nout[at:at] = desired\n\ntry:\n os.makedirs(os.path.dirname(conf) or \".\", exist_ok=True)\n tmp = conf + \".webui-cred.new\"\n with open(tmp, \"w\", encoding=\"utf-8\") as fh:\n fh.write(\"\\n\".join(out) + \"\\n\")\n fh.flush()\n os.fsync(fh.fileno())\n # This hook runs as ROOT; the service runs as PUID:PGID. A root-owned 0600 conf would\n # cost the app the ability to save its own settings, so carry the ownership over.\n try:\n st = os.stat(conf)\n os.chown(tmp, st.st_uid, st.st_gid)\n os.chmod(tmp, stat.S_IMODE(st.st_mode))\n except FileNotFoundError:\n os.chown(tmp, int(os.environ.get(\"PUID\", \"1000\")),\n int(os.environ.get(\"PGID\", \"1000\")))\n os.chmod(tmp, 0o664)\n os.replace(tmp, conf) # atomic: the app never reads a half-written conf\nexcept OSError as exc:\n sys.stderr.write(\"[qbt-webui-cred] cannot write %s: %s\\n\" % (conf, exc))\n raise SystemExit(1)\nraise SystemExit(10)\nPY\n rc=$?\n\n if [ \"$rc\" = 0 ]; then\n log \"conf already holds the durable credential for '$QBT_WEBUI_USERNAME' (no write)\"\n elif [ \"$rc\" = 10 ]; then\n log \"asserted WebUI Username + Password_PBKDF2 into $CONF\"\n # postStart runs CONCURRENTLY with the s6 init chain, so qbittorrent-nox may ALREADY\n # have read the old, credential-less conf and be serving a random temporary password.\n # Restart the SUPERVISED SERVICE (not the pod) so it re-reads what was just written.\n # Nothing to do when the service has not started yet - it will read the new file.\n # Its own counter, NOT the shared `waited` API budget: sharing one counter is how the\n # 2026-09-13 failure spent the entire verification budget before the first login and\n # then fired attempts 2 and 3 with no wait left at all.\n sv=0\n while [ \"$sv\" -lt 60 ] && [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" != \"true\" ]; do\n sv=$((sv + 1)); sleep 1\n done\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"qbittorrent-nox was already running - restarting it to load the credential\"\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\"\n sleep 3\n fi\n else\n log \"FAILED to write the credential into $CONF (helper exit $rc). The WebUI will keep\" >&2\n log \"minting a random password\
|
||||
\ every start and BOTH arrs will break. Fix and restart.\" >&2\n exit 0\n fi\n\n # (3) GROUND TRUTH: the WebUI itself must accept it. Wait for the API to answer at ALL\n # (shared ~120s budget across attempts), then spend exactly ONE login per attempt -\n # at most 3 rejected logins, which stays under qBittorrent's default ban threshold.\n #\n # \"THE API NEVER ANSWERED\" IS A DIFFERENT FAULT FROM \"THE PASSWORD WAS REJECTED\", and\n # conflating them is what made this hook lie once already (see the PORT comment). An\n # unreachable WebUI is reported as unreachable, and the hook stops rather than spending\n # its remaining login attempts to produce two more identical wrong diagnoses.\n api_ok=0\n while [ \"$waited\" -lt 120 ]; do\n code=$(curl -s -o /dev/null -m 3 -w '%{http_code}' \"http://127.0.0.1:$PORT/\" 2>/dev/null)\n if [ -n \"$code\" ] && [ \"$code\" != \"000\" ]; then api_ok=1; break; fi\n waited=$((waited + 1)); sleep 1\n done\n if [ \"$api_ok\" != 1 ]; then\n log \"the WebUI never answered on 127.0.0.1:$PORT within the wait budget, so the\" >&2\n log \"credential in $CONF is UNVERIFIED. This is NOT a rejected password - it was\" >&2\n log \"never submitted. Do not rotate the credential on the strength of this line.\" >&2\n exit 0\n fi\n body=$(curl -sS -m 10 \\\n --data-urlencode \"username=$QBT_WEBUI_USERNAME\" \\\n --data-urlencode \"password=$QBT_WEBUI_PASSWORD\" \\\n \"http://127.0.0.1:$PORT/api/v2/auth/login\" 2>/dev/null)\n if [ \"$body\" = \"Ok.\" ]; then\n log \"VERIFIED: the WebUI accepts the durable credential for '$QBT_WEBUI_USERNAME'\"\n exit 0\n fi\n log \"WebUI rejected the durable credential (login said '$body')\" >&2\n # A rejection when the conf was ALREADY converged (rc=0, nothing written) can only mean\n # the running qbittorrent-nox is serving something it read from an EARLIER conf, so\n # rewriting the same bytes on the next attempt would change nothing. The remedy is to\n # make the process RE-READ the file, so restart the supervised service and let the next\n # attempt re-verify. Without this the retry loop could never change its own outcome.\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"restarting qbittorrent-nox so it re-reads $CONF, then re-verifying\" >&2\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\" >&2\n sleep 3\n fi\ndone\n\nlog \"GAVE UP after $attempt attempts: the WebUI does not accept the credential from git.\" >&2\n# A postStart hook MUST exit 0 - a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), turning a credential fault into an outage.\n# Both failure paths above therefore log LOUDLY and exit 0; the `up` SLI still watches the\n# app, and the arrs' own download-client health check surfaces a broken credential.\nexit 0\n"
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -49,7 +49,7 @@ spec:
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
# Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered
|
||||
# semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
|
||||
@@ -77,6 +77,42 @@ spec:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
# act_runner config file — DELTAS ONLY. act_runner v0.6.1 LoadDefault()
|
||||
# (internal/pkg/config/config.go) unmarshals this file onto a ZERO Config and then fills every
|
||||
# unset field with the same default it uses when there is no file at all, so a file holding
|
||||
# only `runner.capacity` changes exactly that and nothing else. Do NOT paste in the output of
|
||||
# `act_runner generate-config`: that example is NOT the defaults. Its `runner.labels` would
|
||||
# OVERRIDE the registered labels in /data/.runner (daemon.go: config labels win when non-empty)
|
||||
# and repoint ubuntu-latest at a different job image; its `force_pull: true` differs from the
|
||||
# no-file default (false); and its sample `envs` would be injected into every job.
|
||||
#
|
||||
# Using a file also makes act_runner IGNORE the legacy GITEA_RUNNER_CAPACITY / _FILE /
|
||||
# _ENVIRON / _ENV_FILE / GITEA_DEBUG / GITEA_TRACE env vars (deprecated.go). None of them is
|
||||
# set on this Deployment; the GITEA_INSTANCE_URL / _REGISTRATION_TOKEN / _NAME / _LABELS env
|
||||
# below is consumed by the image's run.sh for registration, not by that legacy path.
|
||||
#
|
||||
# A ConfigMap edit alone does NOT restart the runner (it reads the file once at start). If a
|
||||
# future change touches ONLY this ConfigMap, also bump the Deployment's pod template (e.g. the
|
||||
# config-revision annotation below) or the new value stays inert until the next restart.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: act-runner-config
|
||||
namespace: gitea-actions
|
||||
labels:
|
||||
app.kubernetes.io/name: act-runner
|
||||
data:
|
||||
config.yaml: |
|
||||
runner:
|
||||
# Two concurrent jobs (default 1). Measured 2026-11-08: queue wait ~1s normally, but
|
||||
# 29-759s whenever gate / images / renovate runs collide (run 472 waited 759s behind a
|
||||
# renovate burst). Both jobs share the ONE dind daemon below; act_runner names each job's
|
||||
# containers GITEA-ACTIONS-TASK-<task id> and creates a network per job, so they do not
|
||||
# collide. Workflow `concurrency:` groups are enforced by Gitea server-side and are
|
||||
# unaffected: gate.yml's gate-<ref> group still cancels a superseded run on the SAME ref,
|
||||
# and renovate's group still serialises renovate. The dind resources are sized for TWO jobs.
|
||||
capacity: 2
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -95,6 +131,10 @@ spec:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: act-runner
|
||||
annotations:
|
||||
# Bump when act-runner-config changes without any other pod-template change, so the
|
||||
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
|
||||
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
|
||||
spec:
|
||||
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
||||
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
||||
@@ -133,7 +173,7 @@ spec:
|
||||
initContainers:
|
||||
- name: dind
|
||||
restartPolicy: Always # <- this field is what makes it a SIDECAR
|
||||
image: docker:27-dind
|
||||
image: docker:29-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
@@ -146,12 +186,23 @@ spec:
|
||||
mountPath: /certs
|
||||
- name: dind-storage
|
||||
mountPath: /var/lib/docker
|
||||
# Sized for runner.capacity: 2. Every job's containers run UNDER this daemon, so this
|
||||
# container's cgroup carries the whole CI workload. Measured 2026-11-08 (Prometheus,
|
||||
# capacity 1): peak CPU 0.69 core (1m rate, 30d), peak working set 447Mi (30d), 0 OOMs.
|
||||
# cpu request 2: two jobs at today's peak is ~1.4 core, and the gate is being
|
||||
# parallelised, so per-job demand rises. The request is what the CFS share is
|
||||
# weighted by under contention (100m was a 1/20 share); NO cpu limit, deliberately,
|
||||
# so an idle node's spare cores are still usable. k3s-prod-2 (8 CPU) had 2945m
|
||||
# requested; +1.9 core = ~4.85 (61%), and prod-1/prod-3 fit too if it reschedules.
|
||||
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
|
||||
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
|
||||
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
memory: 4Gi
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
@@ -167,7 +218,7 @@ spec:
|
||||
containers:
|
||||
# ---- act_runner: registers with Gitea, dispatches jobs to the DinD daemon ----
|
||||
- name: runner
|
||||
image: gitea/act_runner:0.2.11
|
||||
image: gitea/act_runner:0.6.1
|
||||
# No `command:` override. The image's own entrypoint (/sbin/tini -- /opt/act/run.sh)
|
||||
# registers from env on first boot and runs `act_runner daemon`. The two wait-loops
|
||||
# that used to live here are gone: kubelet now holds this container until the dind
|
||||
@@ -195,23 +246,36 @@ spec:
|
||||
value: /certs/client
|
||||
- name: DOCKER_TLS_VERIFY
|
||||
value: "1"
|
||||
# The image's run.sh passes `--config <this>` to BOTH `act_runner register` and
|
||||
# `act_runner daemon` when CONFIG_FILE is set (act_runner v0.6.1 scripts/run.sh).
|
||||
- name: CONFIG_FILE
|
||||
value: /etc/act_runner/config.yaml
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: certs
|
||||
mountPath: /certs
|
||||
readOnly: true
|
||||
- name: config
|
||||
mountPath: /etc/act_runner
|
||||
readOnly: true
|
||||
# Measured 2026-11-08 at capacity 1: peak CPU 0.24 core, peak working set 314Mi (30d)
|
||||
# against the old 512Mi limit. It holds per-task state, log buffers and the embedded
|
||||
# actions/cache server for every concurrent task, so the limit is doubled for capacity 2.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: act-runner-data
|
||||
- name: certs
|
||||
emptyDir: {}
|
||||
- name: config
|
||||
configMap:
|
||||
name: act-runner-config
|
||||
- name: dind-storage
|
||||
emptyDir: {}
|
||||
@@ -75,7 +75,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: keycloak
|
||||
image: quay.io/keycloak/keycloak
|
||||
image: quay.io/keycloak/keycloak:26.0.6
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: radarr
|
||||
image: lscr.io/linuxserver/radarr:5.21.1
|
||||
image: lscr.io/linuxserver/radarr:5.28.0
|
||||
ports:
|
||||
- containerPort: 7878
|
||||
env:
|
||||
@@ -138,7 +138,7 @@ spec:
|
||||
- name: media
|
||||
mountPath: /media
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: semaphore
|
||||
image: semaphoreui/semaphore:v2.19.10
|
||||
image: semaphoreui/semaphore:v2.19.17
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: sonarr
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15
|
||||
image: lscr.io/linuxserver/sonarr:4.0.20
|
||||
ports:
|
||||
- containerPort: 8989
|
||||
env:
|
||||
@@ -138,7 +138,7 @@ spec:
|
||||
- name: media
|
||||
mountPath: /media
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -75,7 +75,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: seerr
|
||||
image: ghcr.io/seerr-team/seerr:v3.4.1
|
||||
image: ghcr.io/seerr-team/seerr:v3.5.1
|
||||
ports:
|
||||
- containerPort: 5055
|
||||
volumeMounts:
|
||||
|
||||
+52
-5
@@ -10,7 +10,7 @@
|
||||
name: gitea
|
||||
kind: container
|
||||
source:
|
||||
image: gitea/gitea:1.24.3 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
image: gitea/gitea:28.3.0 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
# Headless-install bootstrap (formerly apps/gitea-bootstrap-patch.yaml; now first-class via the
|
||||
# ServiceDefinition env/envFrom/lifecycle fields, schema >=1.2.0). Gitea must come up INSTALLED
|
||||
# without the interactive web wizard and with a FIXED, persistent SECRET_KEY, so all of this is
|
||||
@@ -23,14 +23,43 @@ env:
|
||||
secretKeyRef:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN # correct public clone/redirect URLs for the TLS-terminated host
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
# ROOT_URL is the ONLY source of the instance domain since Gitea 28 (go-gitea/gitea#39400): [server]
|
||||
# DOMAIN is no longer read, and the SSH domain defaults from ROOT_URL too. GITEA__server__DOMAIN was
|
||||
# removed in the 28 upgrade; the stale DOMAIN line environment-to-ini already wrote into app.ini is
|
||||
# inert. Correct public clone/redirect URLs for the TLS-terminated host come from this one value.
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED # Phase 6 CI/CD: declared (not implicit upstream default) so the
|
||||
value: "true" # in-cluster act_runner (ns gitea-actions) registers against it
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL # resolve `uses: <org>/<action>@ref` from github.com (cluster
|
||||
value: github # has internet egress; not air-gapped)
|
||||
# Gitea 28 added run retention (go-gitea/gitea#38855): completed runs, with their jobs, logs and
|
||||
# artifacts, are deleted by the cleanup_action_runs cron after RUN_RETENTION_DAYS. Upstream's default
|
||||
# (400) is kept but DECLARED, so the deletion horizon is visible here rather than implicit. The spec
|
||||
# repo states no CI-history retention requirement; 0 would mean keep forever.
|
||||
- name: GITEA__actions__RUN_RETENTION_DAYS
|
||||
value: "400"
|
||||
# Gitea 28 (go-gitea/gitea#39426) puts webhooks, the OAuth2 avatar fetch and OpenID 2.0 behind an
|
||||
# egress policy: in the default lax mode, PUBLIC targets are allowed and private/loopback/CGNAT
|
||||
# targets need an IP or CIDR entry in [security] ALLOWED_HOST_LIST (a hostname entry is NOT enough
|
||||
# for a private address). DECLARED lax with an EMPTY allow list, deliberately:
|
||||
# * the Keycloak OIDC login source is NOT gated by it. In 28.3.0 the openidConnect provider is
|
||||
# goth's, built with its default HTTP client (services/auth/source/oauth2/providers_openid.go);
|
||||
# egress.NewSecurityPolicy is used only by routers/web/auth/oauth.go (avatar fetch, and
|
||||
# [oauth2_client] UPDATE_AVATAR is off here) and modules/auth/openid (OpenID 2.0, unused).
|
||||
# So keycloak.<domain> resolving to the private Gateway address in-cluster does not matter.
|
||||
# * this instance has no webhooks and no mirrors (measured 2026-11-07), and 1.27 already refused
|
||||
# private webhook targets by default, so nothing that worked before is newly denied.
|
||||
# Adding an entry here would only WIDEN the webhook SSRF surface. Add the narrowest CIDR (e.g. the
|
||||
# Gateway address /32) only if a real private webhook target appears.
|
||||
- name: GITEA__security__EGRESS_MODE
|
||||
value: lax
|
||||
# Gitea 28 (go-gitea/gitea#39400) disables LOCAL self-registration unless DISABLE_REGISTRATION is
|
||||
# explicitly false; 1.27 defaulted it open. Declared true: accounts arrive via Keycloak only.
|
||||
# OAuth2 auto-registration below is unaffected — routers/web/auth/oauth.go gates it on
|
||||
# ALLOW_ONLY_INTERNAL_REGISTRATION and ENABLE_AUTO_REGISTRATION, not on DISABLE_REGISTRATION.
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: "true"
|
||||
- name: GITEA_ADMIN_USERNAME # break-glass local admin creds (consumed by the postStart hook)
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -208,6 +237,19 @@ inClusterClients:
|
||||
namespace: flux-system # matched by NAMESPACE alone — the flux controllers' pod labels belong
|
||||
# to upstream's manifests, not to this platform's contract
|
||||
port: 3000 # ONLY the Gitea HTTP port — the same single port the Gateway allow opens
|
||||
# The PM agent service (platform/pm-agent/). Its dispatcher reads issues and posts comments
|
||||
# over this port, and every worker Job clones, pushes and opens pull requests over it. Like
|
||||
# Flux's source-controller above, it dials the ClusterIP DIRECTLY — deliberately, so the
|
||||
# agent's path to the repository depends on neither public DNS, nor Cloudflare, nor the
|
||||
# shared Gateway, nor ${BASE_DOMAIN}. And exactly as for Flux, that means this namespace's
|
||||
# own default-deny (R-NET-01) DROPS the connection unless it is declared here: without this
|
||||
# entry the dispatcher's egress allow is satisfied and the packet is still discarded at the
|
||||
# Gitea end, which presents as an authentication timeout rather than as a policy denial.
|
||||
# That is the 2026-09-13 lesson documented at length above, applied before it is relearned.
|
||||
- name: pm-agent
|
||||
namespace: pm-agent # matched by NAMESPACE alone, so the dispatcher Deployment and every
|
||||
# ephemeral worker Job are covered by the one declaration
|
||||
port: 3000 # ONLY the Gitea HTTP port
|
||||
auth:
|
||||
mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC" set; R-IDP-02)
|
||||
groups:
|
||||
@@ -220,9 +262,14 @@ auth:
|
||||
# recovery tools (docs/15 "Availability caveat"). No secret is embedded here.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=1.22.0 <2.0.0" # stay on the 1.x stable line
|
||||
range: ">=28.0.0 <29.0.0" # Gitea dropped the "1." prefix at 28.0.0 (successor of 1.27); stay on 28.x
|
||||
cadence: "0 4 * * *"
|
||||
autoRollbackOnFailedHealth: true # R-UPD-03
|
||||
# R-UPD-03. INERT FOR THIS SERVICE, and it must stay that way: this flag renders only an annotation
|
||||
# on a Flux ImagePolicy, and apps/ is rendered --no-image-automation (ci/render-golden.sh), so no
|
||||
# ImagePolicy exists and the Deployment (strategy Recreate) is never auto-reverted. That matters
|
||||
# because a Gitea upgrade (1.27 -> 28 included) MIGRATES THE SQLITE DB ONE-WAY: an image revert onto
|
||||
# a migrated DB is not a rollback. Roll Gitea back by RESTORING THE DATA PVC from its pre-upgrade backup.
|
||||
autoRollbackOnFailedHealth: true
|
||||
observability:
|
||||
metrics: true
|
||||
slis:
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
name: jackett
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/jackett:latest
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3030-ls30
|
||||
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
||||
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
@@ -70,7 +70,8 @@ auth:
|
||||
- ^/api/
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=0.22.0" # stay on the Jackett 0.22.x stable line
|
||||
range: ">=0.22.0" # floor only, no ceiling (0.24.x runs as of 2026-10-30; Jackett has no
|
||||
# major line to hold back from, so renovate.json5 mirrors no cap)
|
||||
cadence: "0 4 * * *"
|
||||
autoRollbackOnFailedHealth: true # R-UPD-03
|
||||
observability:
|
||||
|
||||
@@ -5,8 +5,15 @@
|
||||
# Validates against schemas/servicedefinition.schema.json.
|
||||
name: keycloak
|
||||
kind: container
|
||||
# DOCUMENTATION-ONLY IMAGE. The running server is NOT chosen here: the Keycloak operator
|
||||
# (platform/keycloak/kustomization.yaml, three pinned release URLs) picks it via its
|
||||
# RELATED_IMAGE_KEYCLOAK, and the Keycloak CR has no spec.image. This SD's render,
|
||||
# apps/wave-1/keycloak.yaml, is not listed in apps/wave-1/kustomization.yaml and deploys nothing
|
||||
# (ci/render-golden.sh RENDER_GOLDEN_NOT_ONBOARDED). The tag records the operator's version for
|
||||
# readers; renovate.json5 disables Renovate for this image so it cannot raise PRs against a file
|
||||
# nothing deploys. The Keycloak version moves with the operator — update this tag by hand then.
|
||||
source:
|
||||
image: quay.io/keycloak/keycloak
|
||||
image: quay.io/keycloak/keycloak:26.0.6
|
||||
exposure: public # login/redirect flows for internet-exposed services; hardened (docs/15; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
|
||||
+89
-31
@@ -5,7 +5,7 @@
|
||||
name: pihole
|
||||
kind: container
|
||||
source:
|
||||
image: pihole/pihole
|
||||
image: pihole/pihole:2026.10.1
|
||||
exposure: internal # LAN-only admin UI (docs/13 row 6; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
@@ -85,7 +85,7 @@ env:
|
||||
# Default is LOCAL, which answers only clients on the same subnet as the listening interface. The
|
||||
# pod's interface is on the CLUSTER pod CIDR, so every LAN client would be refused. ALL is
|
||||
# required for a LoadBalancer-fronted resolver; what keeps PiHole from being an OPEN resolver is
|
||||
# the exposure, not this setting — `loadBalancerSourceRanges: [${LAN_CIDR}]` on the DNS Services,
|
||||
# the exposure, not this setting — `loadBalancerSourceRanges: [${LAN_CIDR}, ${WG_CLIENT_CIDR}]` on the DNS Services,
|
||||
# enforced by Cilium in the datapath before load-balancing, plus a CiliumNetworkPolicy that admits
|
||||
# nothing but udp/53 and tcp/53. See apps/wave-0/pihole-dns.yaml.
|
||||
- name: FTLCONF_dns_listeningMode
|
||||
@@ -108,43 +108,101 @@ env:
|
||||
# `loadBalancer.mode: dsr`, which preserves the client address.
|
||||
- name: FTLCONF_dns_rateLimit_count
|
||||
value: "0"
|
||||
# THE SPLIT-HORIZON ANSWERS. A dnsmasq `address=/<domain>/<ip>` entry answers that domain AND
|
||||
# every subdomain of it, and dnsmasq resolves overlapping entries by LONGEST MATCHING DOMAIN —
|
||||
# so the first line is a wildcard for the whole zone and each later line is a specific override.
|
||||
# PRECEDENCE IS THEREFORE STRUCTURAL, not order-dependent: `unifi.${BASE_DOMAIN}` matches both
|
||||
# line 1 and line 2, line 2 is the longer domain, line 2 wins. Rewriting these in any order
|
||||
# changes nothing.
|
||||
# THE LAN ANSWERS — AN EXPLICIT LIST, NOT A ZONE WILDCARD (2026-11-09). A dnsmasq
|
||||
# `address=/<domain>/<ip>` entry answers that domain AND every subdomain of it (on LABEL
|
||||
# boundaries: `pihole.` does not match `pihole-dns.`), and dnsmasq resolves overlapping entries by
|
||||
# LONGEST MATCHING DOMAIN, so the order below is irrelevant. Every name NOT listed here is
|
||||
# FORWARDED to the upstreams above and gets the PUBLIC answer.
|
||||
#
|
||||
# A NEWLY ONBOARDED SERVICE NEEDS NO CHANGE HERE. Every catalog ingressRoute host is
|
||||
# `<label>.${BASE_DOMAIN}` served by the shared Gateway, so the wildcard already answers it with
|
||||
# ${GATEWAY_VIP} — the same guarantee, and the same reasoning, as the `template IN A` block in
|
||||
# platform/coredns/coredns-custom.yaml. Add a line here ONLY for a service that does not live
|
||||
# behind the Gateway: an L4 LoadBalancer on its own pinned VIP. Those are enumerated from the
|
||||
# repo, not from memory — grep `external-dns.alpha.kubernetes.io/hostname` on a Service, which
|
||||
# is the annotation that means "this Service publishes its own hostname at its own address":
|
||||
# apps/vms/unifi/50-service-l4.yaml (unifi -> ${UNIFI_L4_VIP}) and
|
||||
# platform/crafty/service-l4.yaml (crafty-mc -> ${CRAFTY_L4_VIP}). Those two, plus PiHole's own
|
||||
# resolver address, are the complete set of exceptions today.
|
||||
# WHY NO WILDCARD ANY MORE. Until 2026-11-09 the first entry was `address=/${BASE_DOMAIN}/
|
||||
# ${GATEWAY_VIP}`, answering the whole zone. That made sense while every public record was a
|
||||
# DNS-only CNAME to `wan.${BASE_DOMAIN}` (no NAT hairpin on this router, so the LAN NEEDED a
|
||||
# private answer). Since PR #156 every public name is a PROXIED CNAME to the Cloudflare Tunnel
|
||||
# (platform/externaldns/README.md), which works identically from the LAN — so a name that is
|
||||
# public now SHOULD resolve publicly from the LAN too, and the LAN no longer depends on PiHole for
|
||||
# it: a client that bypasses PiHole (DoH browser, phone private DNS, the router's fallback) gets
|
||||
# the same working answer. The wildcard had also become actively misleading: it answered names
|
||||
# that exist nowhere (`wan.` itself resolved to the Gateway VIP on the LAN).
|
||||
#
|
||||
# AAAA needs nothing: dnsmasq answers a AAAA query for an `address=`d name with NODATA
|
||||
# (NOERROR, no answer), so IPv6-capable clients fall back to the A record instead of getting
|
||||
# NXDOMAIN — the same behaviour coredns-custom hand-writes a `template IN AAAA` block for.
|
||||
# WHAT IS LISTED, and why each — enforced by conformance/tests/l0-static/pihole-lan-names:
|
||||
# -> ${GATEWAY_VIP}, LAN-ONLY: every HTTPRoute disowned from public DNS
|
||||
# (`external-dns.alpha.kubernetes.io/controller: none`, i.e. catalog `exposure: internal`,
|
||||
# plus plex and headlamp) — jackett, pihole, plex, qbittorrent, radarr, semaphore, sonarr,
|
||||
# headlamp. These have NO public record and the tunnel 404s them, so THIS LINE IS THEIR ONLY
|
||||
# PATH: drop one and the app is unreachable everywhere. A NEWLY ONBOARDED `internal` SERVICE
|
||||
# MUST BE ADDED HERE (the L0 check fails until it is). A new `public` service needs nothing.
|
||||
# -> ${GATEWAY_VIP}, PUBLIC BUT KEPT ON THE LAN PATH: gitea (git push, LFS, container registry)
|
||||
# and nextcloud (uploads) — Cloudflare's proxy caps a request body at 100 MB and a response
|
||||
# at 100 s, which a large push or upload exceeds; keycloak and auth (oauth2-proxy) — so the
|
||||
# OIDC login of those two LAN-path apps stays on the same path as the apps themselves.
|
||||
# -> their own L4 VIPs: unifi (${UNIFI_L4_VIP}) and crafty-mc (${CRAFTY_L4_VIP}) are not behind
|
||||
# the Gateway (grep `external-dns.alpha.kubernetes.io/hostname` on a Service:
|
||||
# apps/vms/unifi/50-service-l4.yaml, platform/crafty/service-l4.yaml); their public records
|
||||
# are DNS-only A records to the same RFC1918 VIPs, which this router's rebind protection
|
||||
# strips, so the LAN answer is still required. pihole-dns (${PIHOLE_DNS_VIP}) is PiHole's own
|
||||
# resolver address; plex-vip (${PLEX_L4_VIP}) is below.
|
||||
# NOT listed, deliberately, so they resolve via Cloudflare from the LAN as from anywhere: the apex,
|
||||
# grafana, crafty, seerr. In-cluster (platform/coredns/coredns-custom.yaml) and node
|
||||
# (/etc/hosts pins, node_baseline_hosts_pins) resolution are separate and unchanged.
|
||||
#
|
||||
# AAAA: dnsmasq answers a AAAA query for an `address=`d name with NODATA (NOERROR, no answer),
|
||||
# so IPv6-capable clients fall back to the A record instead of getting NXDOMAIN — the same
|
||||
# behaviour coredns-custom hand-writes a `template IN AAAA` block for. Unlisted (proxied) names
|
||||
# now get Cloudflare's real A and AAAA answers.
|
||||
#
|
||||
# Array-valued FTLCONF variables are SEMICOLON-separated. A JSON-looking list is accepted and
|
||||
# then silently wrong: FTL stores `["a","b"]` as ONE array element containing those literal
|
||||
# characters, writes that single bogus line into /etc/pihole/dnsmasq.conf and reports "0 invalid"
|
||||
# while resolution for the zone does not work. Confirmed both ways against the live image.
|
||||
# plex-vip -> ${PLEX_L4_VIP} (added 2026-10-04) is the LAN HALF of Plex's single advertised
|
||||
# hostname. Plex publishes ONE URL to plex.tv for both local and remote clients
|
||||
# (catalog/plex.yaml ADVERTISE_IP): on the LAN this answer points it at the tcp/32400
|
||||
# LoadBalancer, while Cloudflare answers the same name publicly with the WAN address, NATed to
|
||||
# the same VIP. WITHOUT THIS LINE the entry above answers it with ${GATEWAY_VIP} via the zone
|
||||
# wildcard, where NOTHING listens on 32400 — measured 2026-10-04, plex-vip.${BASE_DOMAIN}
|
||||
# resolved to the Gateway VIP and a connect to :32400 there returned http=000. That is a silent
|
||||
# failure: the Plex web UI on plex.${BASE_DOMAIN} keeps working through the Gateway on 443 while
|
||||
# every client that takes the ADVERTISED connection cannot reach the server at all.
|
||||
#
|
||||
# plex-vip -> ${PLEX_L4_VIP} (added 2026-10-04) is the name of the tcp/32400 LoadBalancer
|
||||
# (apps/wave-0/plex-remote.yaml) and the domain of the certificate Plex serves there
|
||||
# (catalog/plex.yaml CERTDOMAIN / customCertificateDomain, Secret `plex-vip-tls`). It is NOT what
|
||||
# Plex advertises any more (ADVERTISE_IP is a resolver-independent plex.direct name since
|
||||
# 2026-10-05) and it has NO public record, so this entry is its only resolution — kept for
|
||||
# reaching the VIP by name with a matching certificate from a LAN shell. Measured 2026-10-04,
|
||||
# before it existed the zone wildcard answered it with ${GATEWAY_VIP}, where nothing listens on
|
||||
# 32400; without it now, the name is NXDOMAIN.
|
||||
#
|
||||
# ── local-ttl=60 — THE SPLIT-HORIZON ANSWER MUST BE CACHEABLE ────────────────────────────────
|
||||
# MEASURED 2026-10-18, diagnosing "DNS flapping between the cluster ingress and the public IP":
|
||||
#
|
||||
# dig home.keithjackson.dev @192.168.2.23 (pihole) -> 192.168.2.20 TTL=0
|
||||
# dig home.keithjackson.dev @1.1.1.1 (public) -> wan.home.keithjackson.dev TTL=300
|
||||
# dig home.keithjackson.dev @192.168.2.1 (router) -> wan.... TTL=153 -> 148 -> 143
|
||||
#
|
||||
# dnsmasq answers `address=` entries with **TTL 0** unless told otherwise, and that is the whole
|
||||
# bug. The SOHO router forwards LAN queries to PiHole and CACHES the result, so:
|
||||
# * the CORRECT answer is UNCACHEABLE (TTL 0) -- the router must re-query PiHole on every
|
||||
# single lookup, which is maximum exposure to any momentary gap; while
|
||||
# * the WRONG answer is CACHEABLE FOR 5 MINUTES -- the instant the router falls back to its
|
||||
# upstream (standard forwarder behaviour when the forward target does not answer) it latches
|
||||
# the public WAN address and serves it to EVERY LAN CLIENT for the public record's 300s TTL.
|
||||
# The decrementing TTL on the router's reply above is the proof it caches rather than forwarding
|
||||
# live. So ONE sub-second PiHole gap -- a pod restart, an L2 announcement lease moving, an ARP
|
||||
# flap, a kured node reboot -- becomes FIVE MINUTES of the whole house resolving to the WAN IP.
|
||||
# That asymmetry, not the gap, is why this is visible as "flapping" rather than a blip.
|
||||
#
|
||||
# 60s is a deliberate floor, not a tuning knob: long enough to ride out a pod restart and a lease
|
||||
# re-election (both seconds), short enough that a genuine VIP change still propagates inside a
|
||||
# minute. These VIPs are pinned by name (`lbipam.cilium.io/ips`) and effectively static, so the
|
||||
# propagation cost is theoretical while the flap it prevents is measured. If flaps persist across
|
||||
# LONGER outages, raising this toward the public record's 300 would cover them -- at the cost of
|
||||
# the same delay on a real change. It does NOT help when PiHole is down longer than the TTL: the
|
||||
# cache expires, the router falls back, and the public answer latches again. That is a real
|
||||
# outage, and the fix for it is PiHole availability, not DNS caching.
|
||||
#
|
||||
# SINCE 2026-11-09 THE LATCH IS MOSTLY HARMLESS. The measurement above predates the tunnel: the
|
||||
# public answer was then `wan.`, unreachable from the LAN. Now the public answer for every
|
||||
# proxied name is Cloudflare, which works from the LAN, and those names are not listed below at
|
||||
# all. A latch still bites only the listed names: gitea/nextcloud/keycloak/auth fall back to
|
||||
# Cloudflare (working, but under its 100 MB / 100 s limits); the LAN-only names and the L4 VIPs
|
||||
# get NXDOMAIN or a stripped answer until the router re-queries PiHole.
|
||||
#
|
||||
# Applies to every `address=` reply below (dnsmasq: local-ttl governs replies made from
|
||||
# configuration, /etc/hosts and DHCP leases), so one directive covers every listed name.
|
||||
- name: FTLCONF_misc_dnsmasq_lines
|
||||
value: "address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}"
|
||||
value: "local-ttl=60;address=/gitea.${BASE_DOMAIN}/${GATEWAY_VIP};address=/nextcloud.${BASE_DOMAIN}/${GATEWAY_VIP};address=/keycloak.${BASE_DOMAIN}/${GATEWAY_VIP};address=/auth.${BASE_DOMAIN}/${GATEWAY_VIP};address=/headlamp.${BASE_DOMAIN}/${GATEWAY_VIP};address=/cve.${BASE_DOMAIN}/${GATEWAY_VIP};address=/jackett.${BASE_DOMAIN}/${GATEWAY_VIP};address=/pihole.${BASE_DOMAIN}/${GATEWAY_VIP};address=/plex.${BASE_DOMAIN}/${GATEWAY_VIP};address=/qbittorrent.${BASE_DOMAIN}/${GATEWAY_VIP};address=/radarr.${BASE_DOMAIN}/${GATEWAY_VIP};address=/semaphore.${BASE_DOMAIN}/${GATEWAY_VIP};address=/sonarr.${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}"
|
||||
# THE ADMIN PASSWORD, previously the ONE operator-set value declared nowhere. Sourced from
|
||||
# apps/wave-0/pihole-bootstrap-secret.sops.yaml (SOPS/age, same wave => same reconcile), which
|
||||
# carries the full rationale and the measurement that found it.
|
||||
|
||||
+10
-3
@@ -12,7 +12,7 @@
|
||||
name: plex
|
||||
kind: container
|
||||
source:
|
||||
image: plexinc/pms-docker:latest
|
||||
image: plexinc/pms-docker:1.43.6.11085-d1e52e169
|
||||
# plexinc/pms-docker env. Rendered VERBATIM onto the app container by render.py (schema env list).
|
||||
env:
|
||||
# PLEX_UID/PLEX_GID: the image boots as root (s6 PID1), chowns /config to this uid/gid, then drops
|
||||
@@ -347,11 +347,18 @@ lifecycle:
|
||||
# `runbooks/plex-quicksync.md` is that procedure, plus how to verify it took.
|
||||
hardware:
|
||||
gpu: intel-quicksync
|
||||
exposure: public # edge port-forward + public DNS; clients + Seerr reach it (docs/09; R-DNS/R-NET)
|
||||
# EXPOSURE: internal (operator decision 2026-11-08, Cloudflare Tunnel cut-over). The plex.${BASE_DOMAIN}
|
||||
# web UI is LAN/VPN-only: `internal` renders `external-dns ... controller: none` on the HTTPRoute, so
|
||||
# external-dns DELETES the public plex record and its TXT, and platform/cloudflared/config.yaml maps the
|
||||
# name to http_status:404 so it can never be served through the tunnel either. REMOTE STREAMING IS NOT
|
||||
# AFFECTED: it never used this name. It rides plex.tv + the advertised plex.direct URL (ADVERTISE_IP
|
||||
# above) to the raw tcp/32400 L4 VIP (${PLEX_L4_VIP}, apps/wave-0/plex-remote.yaml) via the WAN
|
||||
# tcp/32400 forward — none of which this field touches. See runbooks/plex-remote-access.md.
|
||||
exposure: internal # web UI LAN/VPN-only; remote streaming = plex.tv + WAN tcp/32400 (docs/09; R-DNS/R-NET)
|
||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||
storage:
|
||||
- name: config
|
||||
size: 8Gi
|
||||
size: 16Gi
|
||||
class: longhorn # Plex DB/metadata: replicated app state on the 3-replica tier (docs/04; R-STATE-01/02)
|
||||
mountPath: /config # plexinc/pms-docker writes its identity/claim + "Library" DB under /config, NOT the
|
||||
# renderer default /data/config. Without this the PVC sat unused and Plex wrote to the
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
name: qbittorrent
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
image: lscr.io/linuxserver/qbittorrent:5.3.0 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING FOR THE BULK MOUNT, not cosmetic — exactly as on sonarr/radarr. Measured on prod
|
||||
# 2026-09-12 BEFORE this change: `id` inside the container reported the qbittorrent-nox process as
|
||||
@@ -359,7 +359,7 @@ storage:
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE DOWNLOADS AREA — ONE mount of the SAME export sonarr/radarr/plex already consume
|
||||
# (192.168.2.102:/mnt/Storage/plex), narrowed by subPath to the `downloads` subtree and presented at
|
||||
# (192.168.2.30:/mnt/Storage/plex), narrowed by subPath to the `downloads` subtree and presented at
|
||||
# /downloads, which is precisely where this app is ALREADY configured to write.
|
||||
#
|
||||
# WHY subPath: downloads AND NOT THE EXPORT ROOT AT /media (the shape sonarr/radarr use):
|
||||
@@ -434,7 +434,7 @@ dependencies: [] # default-deny: reaches only DNS; arr reach it n
|
||||
# for weeks and neither has any CNP naming the NFS server or port 2049
|
||||
# (grep apps/wave-1/sonarr.yaml: none). The SAME fact is what keeps the VPN
|
||||
# egress gateway off this traffic (its selector is a podSelector), and
|
||||
# 192.168.2.102 is inside the 192.168.0.0/16 excludedCIDR besides — see
|
||||
# 192.168.2.30 is inside the 192.168.0.0/16 excludedCIDR besides — see
|
||||
# platform/vpn-egress/egress-steering.yaml, "BULK NFS".
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
image: lscr.io/linuxserver/radarr:5.28.0 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
|
||||
+15
-8
@@ -129,7 +129,7 @@ AUTH_MODES = ("oidc", "forward-auth", "native", "none")
|
||||
# the sidecar authenticates the browser against Keycloak (homelab realm, the app's auth.groups)
|
||||
# and only THEN proxies to the co-located app UI at http://127.0.0.1:<uiport>. The gate is IN
|
||||
# the data path (fail-closed: no valid session in an authorized group => no upstream; R-IDP-02).
|
||||
OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
|
||||
OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.15.6"
|
||||
OAUTH2_PROXY_PORT = 4180
|
||||
KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
|
||||
|
||||
@@ -156,11 +156,17 @@ OAUTH2_COOKIE_REFRESH = "2m30s"
|
||||
# ready endpoints. The flag is confirmed present in the PINNED v7.8.1 binary (`--help`),
|
||||
# which matters because the shared proxy is on v7.15.3 and could not have vouched for it.
|
||||
#
|
||||
# THIS IS THE PREREQUISITE ONLY, AND IT IS NOT YET SCRAPED. A container port is not a
|
||||
# Service port: these sidecars' Services expose the app port and :4180, and each namespace's
|
||||
# default-deny baseline has `ingress: []`, so Prometheus can neither resolve nor reach this
|
||||
# port until a Service port (or PodMonitor), a monitoring-ingress allow and a ServiceMonitor
|
||||
# are added. Do not read this flag as "metrics are working".
|
||||
# THE SCRAPE WORKS AS OF 2026-10-05 — but it took THREE independent pieces, and this flag is
|
||||
# only the first. A container port is not a Service port, and each namespace's default-deny
|
||||
# baseline denies all ingress, so the flag alone bought nothing for 63.7h:
|
||||
# 1. this flag + a named `metrics` container port (PR #28)
|
||||
# 2. a monitor that targets POD ports, not Service ports — platform/oauth2-proxy/podmonitor.yaml
|
||||
# is a PodMonitor for exactly that reason (no Service exposes 44180)
|
||||
# 3. an ingress allowance from ns monitoring (PR #49, r_monitoring_ingress_policy below)
|
||||
# Verified on prod 2026-10-05: all 6 targets up == 1, scraping in 1-6ms.
|
||||
# STILL do not read this flag alone as "metrics are working" — 1 without 3 is precisely the state
|
||||
# that produced 5 x TargetDown for 63.7h. If you add a sidecar metrics port anywhere else, check
|
||||
# that r_monitoring_ingress_policy covers it.
|
||||
OAUTH2_PROXY_METRICS_PORT = 44180
|
||||
|
||||
# --- Hardware capability: `hardware.gpu: intel-quicksync` ------------------------------
|
||||
@@ -1362,8 +1368,9 @@ def _oauth2_sidecar_container(s: dict) -> dict:
|
||||
f"--cookie-refresh={OAUTH2_COOKIE_REFRESH}",
|
||||
"--pass-access-token=true",
|
||||
"--set-xauthrequest=true",
|
||||
# Start the /metrics listener. Prerequisite only — not yet reachable by Prometheus;
|
||||
# see OAUTH2_PROXY_METRICS_PORT above for exactly what is still missing.
|
||||
# Start the /metrics listener. Scraped as of 2026-10-05, but only because
|
||||
# r_monitoring_ingress_policy also opens this port to ns monitoring — the flag alone was
|
||||
# inert for 63.7h. See OAUTH2_PROXY_METRICS_PORT above for the three pieces required.
|
||||
f"--metrics-address=0.0.0.0:{OAUTH2_PROXY_METRICS_PORT}",
|
||||
]
|
||||
secret = f"{s['name']}-oauth2-proxy"
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
name: seerr
|
||||
kind: container
|
||||
source:
|
||||
image: ghcr.io/seerr-team/seerr:v3.4.1 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
|
||||
image: ghcr.io/seerr-team/seerr:v3.5.1 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
|
||||
# The ghcr.io/seerr-team/seerr image runs as the non-root user `node:node` (UID 1000 / GID 1000 —
|
||||
# the standard node base-image user; confirmed from the image config's User field). On boot the app
|
||||
# creates /app/config/logs and writes its settings/DB under the mounted /app/config PVC. A fresh
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
name: semaphore
|
||||
kind: container
|
||||
source:
|
||||
image: semaphoreui/semaphore:v2.19.10 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
image: semaphoreui/semaphore:v2.19.17 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
env:
|
||||
# --- PostgreSQL connection (the CNPG-DB pattern). DIALECT/HOST/PORT/DB/USER are literals; PASS is
|
||||
# sourced from the in-namespace SOPS secret (the CNPG -app secret lives in ns semaphore-db,
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
name: sonarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/sonarr:4.0.15 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
image: lscr.io/linuxserver/sonarr:4.0.20 # pinned stable v4 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
|
||||
+1
-1
@@ -99,7 +99,7 @@ caller that must distinguish them invokes `./ci/gate.sh` directly.
|
||||
|
||||
| stage | a skip there means | verdict |
|
||||
|---|---|---|
|
||||
| `lint`, `l0`, `render-golden`, `l1` | a **pinned tool is absent** or a **wired seam went unwired** — it *could not run* | **BLOCKED** under strict skips (a CI runner: the toolchain is installed for us by `.gitea/workflows/scripts/install-tools.sh`, so a skip is a defect); **UNDETERMINED** otherwise (a workstation missing a tool is a real local condition — still non-zero, still not a pass; the remedy is `infra/devcontainer/dev.sh`, which carries the pinned toolchain) |
|
||||
| `lint`, `l0`, `render-golden`, `l1` | a **pinned tool is absent** or a **wired seam went unwired** — it *could not run* | **BLOCKED** under strict skips (a CI runner: the toolchain is the job image — the CI-built toolbox, `.gitea/workflows/gate.yml` `container:` — so a skip is a defect); **UNDETERMINED** otherwise (a workstation missing a tool is a real local condition — still non-zero, still not a pass; the remedy is `infra/devcontainer/dev.sh`, which carries the pinned toolchain) |
|
||||
| `spec-immutability` (R-TEST-06) | the sibling spec repo is not checked out, so the spec **could not be verified** | **UNDETERMINED** — unverified is unverified. Deliberately *not* upgraded to BLOCKED by strict skips: the live runner does not check the spec repo out at all, so blocking would leave the only enforced status check permanently red for a reason no PR could clear. It is therefore the **one** skip whose non-answer a caller may knowingly tolerate, and `.gitea/workflows/gate.yml` does so in one visible line. `REQUIRE_SPEC_VERIFICATION=true` makes it a hard block once that checkout is wired. Locally: check the spec repo out beside this one, or set `SPEC_REPO` (`dev.sh` mounts a sibling automatically). |
|
||||
| `decisions-log` | **nothing** — this stage has no skip path. It reads only files in the tree under test and needs no tool and no cluster, so there is no environment in which it "could not run" | a reported skip is a **defect in the stage** and BLOCKS. The stage fails when the frozen `DECISIONS-LOG.md` is modified (which silently re-creates, for every *other* open branch, the tail-hunk conflict that made four of five PRs unmergeable on 2026-09-29) or when a `decisions/` entry is malformed. There is deliberately **no** `REQUIRE_DECISIONS_LOG` knob: a guard with an off switch is not a guard. See `decisions/README.md`. |
|
||||
| `conformance` (L2/L3) | `CLUSTER_AVAILABLE!=true` — the environment **declared** the capability absent (documented Phase-0 behaviour) | **does not deny a PASS**: a declared-inapplicable stage was never *supposed* to run. It is NAMED in the PASS headline and step summary, the touched requirements it would have covered are reported UNVERIFIED, and `REQUIRE_CLUSTER_TESTS=true` (Phase 1+) turns that into a block. A skip **despite** `CLUSTER_AVAILABLE=true` is a defect and BLOCKS. |
|
||||
|
||||
+3
-3
@@ -46,8 +46,8 @@
|
||||
# WHICH SKIPS BLOCK, AND WHY (per stage — a skip is not one thing):
|
||||
#
|
||||
# lint / l0 / render-golden / l1 A skip here means a PINNED tool is absent or a wired
|
||||
# seam went unwired. On a CI runner the pinned toolchain is installed for us by
|
||||
# .gitea/workflows/scripts/install-tools.sh, so a skip there is a DEFECT, not an
|
||||
# seam went unwired. On a CI runner the pinned toolchain is the job image itself (the
|
||||
# CI-built toolbox, .gitea/workflows/gate.yml), so a skip there is a DEFECT, not an
|
||||
# environment: it BLOCKS. On a workstation a missing tool is a real local condition, so
|
||||
# it degrades to UNDETERMINED (still non-zero, still never a pass). See strict_skips()
|
||||
# in ci/lib.sh for how CI is detected and why misdetecting it cannot fail open.
|
||||
@@ -257,7 +257,7 @@ for s in lint l0 render-golden l1; do
|
||||
skip)
|
||||
if strict_skips; then
|
||||
err "${s}: skipped (unwired seam / missing tool) under STRICT skips, where the pinned"
|
||||
err " toolchain is installed for us (.gitea/workflows/scripts/install-tools.sh) and every"
|
||||
err " toolchain is baked into the job image (.gitea/workflows/gate.yml container:) and every"
|
||||
err " seam is on main — so this is a DEFECT, not an environment -> blocks merge"
|
||||
summary "- ${s}: **SKIPPED under strict skips** (unwired seam / missing tool) — blocks merge"
|
||||
BLOCK=1
|
||||
|
||||
@@ -97,9 +97,9 @@ cluster_available() { truthy "${CLUSTER_AVAILABLE:-false}"; }
|
||||
# --- Skip strictness: is this a CI runner, or a workstation? -------------------
|
||||
# A SKIPPED stage is never green anywhere (HARD RULE 4), but its SEVERITY differs:
|
||||
#
|
||||
# CI the pinned toolchain is installed by .gitea/workflows/scripts/install-tools.sh
|
||||
# before the gate runs, and every seam is on `main`. A stage that skips here is a
|
||||
# DEFECT (a tool dropped from the installer, a seam that went unwired) -> BLOCK.
|
||||
# CI the pinned toolchain IS the job image (the CI-built toolbox; gate.yml `container:`),
|
||||
# and every seam is on `main`. A stage that skips here is a
|
||||
# DEFECT (a tool dropped from the image, a seam that went unwired) -> BLOCK.
|
||||
# local a missing pinned tool is a real, ordinary local condition -> the gate reports
|
||||
# UNDETERMINED (exit 3): not green, not a pass, but distinguishable from a red.
|
||||
#
|
||||
|
||||
+69
-2
@@ -25,5 +25,72 @@ if [ "${#files[@]}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${files[@]}"
|
||||
ok "yaml/lint passed (${#files[@]} files)"
|
||||
# yamllint is single-threaded, so the file list is split into LINT_JOBS CONTIGUOUS slices of the
|
||||
# sorted list, one yamllint per slice, run concurrently. Each slice's output goes to its own file
|
||||
# and the files are printed in slice order afterwards: the log is the same text, in the same
|
||||
# file order, as one serial `yamllint "${files[@]}"`, and a finding is never interleaved with
|
||||
# another process's output (yamllint's standard format puts the FILE NAME on its own line above
|
||||
# that file's `line:col` findings, so interleaving would misattribute them). The stage's exit
|
||||
# code is the highest any slice returned (yamllint: 1 = error, 2 = warning under strict), so a
|
||||
# single failing slice fails the stage exactly as the serial call did. LINT_JOBS=1 is serial.
|
||||
lint_jobs="${LINT_JOBS:-$(n="$(nproc 2>/dev/null || echo 1)"; [ "$n" -gt 8 ] && n=8; echo "$n")}"
|
||||
case "$lint_jobs" in ''|*[!0-9]*|0) err "LINT_JOBS must be a positive integer, got '${lint_jobs}'"; exit 2 ;; esac
|
||||
[ "$lint_jobs" -gt "${#files[@]}" ] && lint_jobs="${#files[@]}"
|
||||
lint_tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$lint_tmp"' EXIT
|
||||
per=$(( (${#files[@]} + lint_jobs - 1) / lint_jobs ))
|
||||
pids=()
|
||||
for (( i = 0; i < lint_jobs; i++ )); do
|
||||
slice=("${files[@]:$(( i * per )):${per}}")
|
||||
[ "${#slice[@]}" -gt 0 ] || break
|
||||
( rc=0
|
||||
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${slice[@]}" \
|
||||
>"${lint_tmp}/${i}.out" 2>&1 || rc=$?
|
||||
echo "$rc" >"${lint_tmp}/${i}.rc" ) &
|
||||
pids+=("$!")
|
||||
done
|
||||
wait "${pids[@]}" || true
|
||||
lint_rc=0
|
||||
for (( i = 0; i < ${#pids[@]}; i++ )); do
|
||||
cat "${lint_tmp}/${i}.out"
|
||||
# A slice that left no rc file died before yamllint returned: that is a failure, not a pass.
|
||||
slice_rc="$(cat "${lint_tmp}/${i}.rc" 2>/dev/null || true)"
|
||||
case "$slice_rc" in ''|*[!0-9]*) slice_rc=1 ;; esac
|
||||
[ "$slice_rc" -gt "$lint_rc" ] && lint_rc="$slice_rc"
|
||||
done
|
||||
if [ "$lint_rc" -ne 0 ]; then
|
||||
err "yaml/lint failed (yamllint exit ${lint_rc}; ${#files[@]} files in ${#pids[@]} parallel slice(s))"
|
||||
exit "$lint_rc"
|
||||
fi
|
||||
ok "yaml/lint passed (${#files[@]} files, ${#pids[@]} parallel slice(s))"
|
||||
|
||||
# A workflow's `configurationFile:` must name a file that exists. Renovate's did not after #79
|
||||
# renamed renovate.json -> renovate.json5, and every scheduled run failed with ENOENT for 11 days
|
||||
# while the gate stayed green. Valid YAML, so yamllint cannot see it.
|
||||
missing=0
|
||||
while IFS=: read -r wf _ value; do
|
||||
value="${value#*configurationFile:}"
|
||||
value="$(echo "$value" | sed -E 's/#.*//; s/^[[:space:]"'\'']+//; s/[[:space:]"'\'']+$//')"
|
||||
if [ -n "$value" ] && [ ! -e "$value" ]; then
|
||||
err "${wf}: configurationFile '${value}' does not exist in the repo"
|
||||
missing=1
|
||||
fi
|
||||
done < <(grep -nE '^[[:space:]]*configurationFile:' .gitea/workflows/*.y*ml 2>/dev/null || true)
|
||||
[ "$missing" -eq 0 ] || exit 1
|
||||
ok "workflow configurationFile references resolve"
|
||||
|
||||
# A workflow must not use an action that needs a Docker daemon: jobs on this runner get NONE, by
|
||||
# decision (decisions/2026-10-29-image-pipeline.md; act_runner maps docker_host to `-`). Such an
|
||||
# action is valid YAML and passes every other stage, then dies at runtime with
|
||||
# "failed to connect to the docker API at unix:///var/run/docker.sock". renovatebot/github-action
|
||||
# did exactly that on every Renovate run until 2026-10-30 (run 343). Run the tool's image as the
|
||||
# job `container:` instead (see .gitea/workflows/renovate.yaml). Container (`docker://`) actions
|
||||
# and Docker's own daemon-bound actions are rejected for the same reason. Comments are ignored.
|
||||
daemon_uses='uses:[[:space:]]*["'\'']?(docker://|renovatebot/github-action|docker/(build-push|setup-buildx|setup-qemu|login)-action)'
|
||||
bad=0
|
||||
while IFS= read -r hit; do
|
||||
err "${hit%%:*}: needs a Docker daemon, which jobs here do not get: $(echo "${hit#*:*:}" | sed -E 's/^[[:space:]]+//')"
|
||||
bad=1
|
||||
done < <(grep -HnE "^[^#]*${daemon_uses}" .gitea/workflows/*.y*ml 2>/dev/null || true)
|
||||
[ "$bad" -eq 0 ] || exit 1
|
||||
ok "no workflow uses a Docker-daemon-bound action"
|
||||
@@ -0,0 +1,490 @@
|
||||
#!/usr/bin/env python3
|
||||
# =============================================================================
|
||||
# ci/render-keycloak-passkey-policy.py
|
||||
#
|
||||
# Render platform/keycloak/passkey-desired-state.yaml FROM
|
||||
# platform/keycloak/keycloak-realm-import.yaml.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# ---------------
|
||||
# The Keycloak realm import is a ONE-SHOT that fails OPEN: once it rests at
|
||||
# Done=True / Started=False it never re-runs. Measured on prod 2026-10-18, hours
|
||||
# after the passkey declaration merged:
|
||||
#
|
||||
# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath=...status
|
||||
# Done=True Started=False HasErrors=False
|
||||
#
|
||||
# So the WebAuthn passwordless policy and the restructured `browser-mfa` flow that
|
||||
# keycloak-realm-import.yaml declares are INERT. They are in git, Flux applied
|
||||
# them, and they reach the realm never. platform/keycloak/passkey-reconciler.yaml
|
||||
# is the level-triggered loop that closes that; this renderer supplies the DESIRED
|
||||
# STATE it converges towards.
|
||||
#
|
||||
# THE ONE VALUE THAT IS NOT RENDERED: `bind-browser-flow`, git's switch for whether the
|
||||
# reconciler may BIND its verified candidate browser flow (one atomic PUT of the
|
||||
# realm's browserFlow). It is an operator DECISION -- "I have tested the candidate
|
||||
# flow in a browser; make it the login path" -- not a fact derivable from the realm
|
||||
# import, so it cannot be generated. It is instead CARRIED THROUGH from the committed
|
||||
# file: --write preserves it, --check accepts any value of it that is exactly the
|
||||
# string "true" or "false" and rejects everything else (an unquoted YAML true is a
|
||||
# bool, which a ConfigMap rejects at apply time -- failing the whole platform-config
|
||||
# Kustomization -- so it is refused HERE, at merge time). An absent file or key
|
||||
# renders as "false". Going live is therefore a one-line PR flipping that line.
|
||||
#
|
||||
# THE CONSTRAINT THAT SHAPES IT: the reconciler pod runs with
|
||||
# automountServiceAccountToken: false, exactly like its client-secret sibling. It
|
||||
# has NO Kubernetes API access at all, by design (R-SEC-02) — so it cannot read the
|
||||
# KeycloakRealmImport CR to learn what the realm is supposed to look like. Its input
|
||||
# must be MOUNTED. Hence a ConfigMap.
|
||||
#
|
||||
# THAT IS ALSO WHY THIS IS A RENDERER AND NOT A HAND-WRITTEN ConfigMap. A
|
||||
# hand-authored copy of the policy would be a SECOND SOURCE OF TRUTH for the RP ID,
|
||||
# the user-verification requirement and the whole flow tree — free to disagree with
|
||||
# the realm import. "Two declarations that can disagree" is the same shape as the
|
||||
# bug this whole effort is closing. So the ConfigMap is GENERATED from the realm
|
||||
# import and the generation is GATED:
|
||||
# conformance/tests/l0-static/keycloak-passkey-reconciler/check.sh runs this script
|
||||
# with --check on every merge, and the merge fails if the committed file is not what
|
||||
# this renderer produces. Same render==golden idiom as
|
||||
# ci/render-keycloak-client-urls.py, catalog/render.py and ci/render-golden.sh.
|
||||
#
|
||||
# STRINGS ARE COPIED VERBATIM, NEVER RE-DERIVED. webAuthnPolicyPasswordlessRpId is
|
||||
# written in the realm import as the Flux substitution token ${BASE_DOMAIN}, and it
|
||||
# is carried through into the ConfigMap CHARACTER FOR CHARACTER. Both files are
|
||||
# rendered by the SAME Flux Kustomization (./platform-config, whose
|
||||
# postBuild.substituteFrom reads cluster-vars), so both are substituted in the same
|
||||
# pass with the same value and cannot disagree about what the base domain is. This
|
||||
# renderer never parses, splits or reconstructs a hostname and never reads
|
||||
# cluster-vars — there is no second place the domain is decided.
|
||||
#
|
||||
# THE ONE THING THIS RENDERER DERIVES RATHER THAN COPIES
|
||||
# -----------------------------------------------------
|
||||
# The enrolment required action, `webauthn-register-passwordless`. It is
|
||||
# deliberately NOT declared in the realm import, and the import's own header
|
||||
# explains at length why declaring it there would be actively DANGEROUS: Keycloak
|
||||
# 26.0.6 treats `requiredActions` in a realm representation as REPLACE, not merge,
|
||||
# so declaring one action imports exactly that action plus delete_account and skips
|
||||
# DefaultRequiredActions.addActions entirely — deleting CONFIGURE_TOTP,
|
||||
# UPDATE_PASSWORD, VERIFY_EMAIL and seven more. Losing CONFIGURE_TOTP is a hard
|
||||
# lock-out of every platform-admin, because auth-otp-form in the browser flow still
|
||||
# demands a TOTP they could then never enrol.
|
||||
#
|
||||
# So the required action cannot be copied from the import, because it is correctly
|
||||
# absent from it. It is instead DERIVED, as a consequence of what the import DOES
|
||||
# declare:
|
||||
#
|
||||
# IF the declared browser flow contains a `webauthn-authenticator-passwordless`
|
||||
# execution, THEN passkey logins are offered, THEREFORE the enrolment action must
|
||||
# be registered and enabled — and NOT a defaultAction, because the import's whole
|
||||
# stated posture is "additive and optional" and a defaultAction forces enrolment
|
||||
# on every user at next login.
|
||||
#
|
||||
# This is a derivation, not a second declaration: there is no value here that a
|
||||
# human could edit into disagreement with the import. If the import stops declaring
|
||||
# the passkey execution, this renderer stops emitting the required action (and
|
||||
# fails, because then the whole desired state is empty of passkey intent).
|
||||
#
|
||||
# WHAT IS DELIBERATELY NOT EXPORTED
|
||||
# * the TWO-FACTOR webAuthnPolicy* family. Nothing in this realm uses the
|
||||
# `webauthn-authenticator` two-factor authenticator; the passwordless
|
||||
# authenticator reads getWebAuthnPolicyPasswordless() only, so a policy written
|
||||
# into the two-factor family is a silent no-op. Exporting it would hand the
|
||||
# reconciler a write surface with no purpose.
|
||||
# * `clients`, `users`, `groups`, `roles`. Client secrets and client URLs are the
|
||||
# SIBLING reconciler's surface (platform/keycloak/client-secret-reconciler.yaml),
|
||||
# and users/groups/roles are nobody's — prod's real user and the media-users /
|
||||
# platform-admins / viewers groups must survive every run of everything.
|
||||
# * execution `priority`. It is consumed here to ORDER the exported executions
|
||||
# (Keycloak returns them in priority order), but the numbers themselves are not
|
||||
# exported: the reconciler compares the resulting SEQUENCE, which is what
|
||||
# actually determines behaviour. Exporting the integers would make a
|
||||
# priority renumbering that changes no ordering read as drift.
|
||||
#
|
||||
# USAGE
|
||||
# python3 ci/render-keycloak-passkey-policy.py # print the rendered file
|
||||
# python3 ci/render-keycloak-passkey-policy.py --write # write it (regenerate)
|
||||
# python3 ci/render-keycloak-passkey-policy.py --check # exit 1 if it has drifted
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("FATAL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
|
||||
OUT_FILE = "platform/keycloak/passkey-desired-state.yaml"
|
||||
DATA_KEY = "passkey-desired-state.json"
|
||||
CONFIGMAP_NAME = "keycloak-passkey-desired-state"
|
||||
NAMESPACE = "keycloak"
|
||||
|
||||
# The realm keys this renderer exports, and therefore the ONLY realm keys the
|
||||
# reconciler may write. Every one is a webAuthnPolicyPasswordless* field. Kept in
|
||||
# lockstep with POLICY_FIELDS in the reconciler script; the L0 check asserts the two
|
||||
# tuples are identical, so they cannot drift apart silently.
|
||||
POLICY_STR_FIELDS = (
|
||||
"webAuthnPolicyPasswordlessRpId",
|
||||
"webAuthnPolicyPasswordlessRpEntityName",
|
||||
"webAuthnPolicyPasswordlessRequireResidentKey",
|
||||
"webAuthnPolicyPasswordlessUserVerificationRequirement",
|
||||
"webAuthnPolicyPasswordlessAuthenticatorAttachment",
|
||||
"webAuthnPolicyPasswordlessAttestationConveyancePreference",
|
||||
)
|
||||
POLICY_LIST_FIELDS = (
|
||||
"webAuthnPolicyPasswordlessSignatureAlgorithms",
|
||||
"webAuthnPolicyPasswordlessAcceptableAaguids",
|
||||
)
|
||||
POLICY_INT_FIELDS = ("webAuthnPolicyPasswordlessCreateTimeout",)
|
||||
POLICY_BOOL_FIELDS = ("webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister",)
|
||||
POLICY_FIELDS = (POLICY_STR_FIELDS + POLICY_LIST_FIELDS + POLICY_INT_FIELDS
|
||||
+ POLICY_BOOL_FIELDS)
|
||||
|
||||
# The field Keycloak 26.0.6 gates the WHOLE passwordless policy on
|
||||
# (DefaultExportImportManager.updateRealm: the policy is applied only when
|
||||
# RpEntityName is non-null and non-empty). A desired state without it would produce
|
||||
# an accepted PUT that changes nothing, so refuse to render one.
|
||||
RP_ENTITY_NAME = "webAuthnPolicyPasswordlessRpEntityName"
|
||||
|
||||
# The execution whose presence is what makes passkey enrolment necessary.
|
||||
PASSKEY_AUTHENTICATOR = "webauthn-authenticator-passwordless"
|
||||
ENROL_ACTION = "webauthn-register-passwordless"
|
||||
|
||||
|
||||
BIND_KEY = "bind-browser-flow"
|
||||
BIND_VALUES = ("true", "false")
|
||||
|
||||
|
||||
def committed_bind(root: Path) -> str:
|
||||
"""The operator's bind-browser-flow value from the COMMITTED file, or "false".
|
||||
|
||||
This is the only input read from the output file, and it is validated, not trusted:
|
||||
anything other than the exact strings "true" / "false" is a hard failure."""
|
||||
target = root / OUT_FILE
|
||||
if not target.is_file():
|
||||
return "false"
|
||||
docs = [d for d in yaml.safe_load_all(target.read_text(encoding="utf-8")) if d]
|
||||
cm = next((d for d in docs if d.get("kind") == "ConfigMap"), None) or {}
|
||||
data = cm.get("data") or {}
|
||||
if BIND_KEY not in data:
|
||||
return "false"
|
||||
v = data[BIND_KEY]
|
||||
if not isinstance(v, str) or v not in BIND_VALUES:
|
||||
raise SystemExit(
|
||||
"FATAL: %s data.%s is %r. It must be exactly the QUOTED string \"true\" or "
|
||||
"\"false\": an unquoted true/false is a YAML boolean, which a ConfigMap rejects "
|
||||
"at apply time and which would fail the whole platform-config Kustomization"
|
||||
% (OUT_FILE, BIND_KEY, v))
|
||||
return v
|
||||
|
||||
|
||||
def render(root: Path) -> str:
|
||||
"""Return the full text of passkey-desired-state.yaml as it should be committed."""
|
||||
docs = list(yaml.safe_load_all((root / IMPORT_FILE).read_text()))
|
||||
imp = next((d for d in docs if d and d.get("kind") == "KeycloakRealmImport"), None)
|
||||
if imp is None:
|
||||
raise SystemExit("FATAL: no KeycloakRealmImport document in %s" % IMPORT_FILE)
|
||||
realm = (imp.get("spec") or {}).get("realm") or {}
|
||||
realm_name = realm.get("realm")
|
||||
if not isinstance(realm_name, str) or not realm_name:
|
||||
raise SystemExit("FATAL: %s declares no spec.realm.realm name" % IMPORT_FILE)
|
||||
|
||||
# ---- the passwordless policy, copied VERBATIM ----------------------------------
|
||||
policy: dict[str, object] = {}
|
||||
for f in POLICY_STR_FIELDS:
|
||||
if f not in realm:
|
||||
continue # not declared => NOT MANAGED
|
||||
v = realm[f]
|
||||
# A bare ${BASE_DOMAIN} parses as a plain string; an int or bool here would be
|
||||
# a YAML quoting mistake that must not be silently stringified.
|
||||
if not isinstance(v, str):
|
||||
raise SystemExit("FATAL: %s realm field %s is not a string (got %r) — "
|
||||
"quote it in the realm import" % (IMPORT_FILE, f, v))
|
||||
policy[f] = v
|
||||
for f in POLICY_LIST_FIELDS:
|
||||
if f not in realm:
|
||||
continue
|
||||
v = realm[f]
|
||||
if not isinstance(v, list) or not all(isinstance(x, str) for x in v):
|
||||
raise SystemExit("FATAL: %s realm field %s is not a list of strings"
|
||||
% (IMPORT_FILE, f))
|
||||
policy[f] = list(v) # verbatim, order as authored
|
||||
for f in POLICY_INT_FIELDS:
|
||||
if f not in realm:
|
||||
continue
|
||||
v = realm[f]
|
||||
if not isinstance(v, int) or isinstance(v, bool):
|
||||
raise SystemExit("FATAL: %s realm field %s is not an integer"
|
||||
% (IMPORT_FILE, f))
|
||||
policy[f] = v
|
||||
for f in POLICY_BOOL_FIELDS:
|
||||
if f not in realm:
|
||||
continue
|
||||
v = realm[f]
|
||||
if not isinstance(v, bool):
|
||||
raise SystemExit("FATAL: %s realm field %s is not a boolean"
|
||||
% (IMPORT_FILE, f))
|
||||
policy[f] = v
|
||||
|
||||
if not policy:
|
||||
# Fail closed. An empty policy would make the reconciler believe every field is
|
||||
# unmanaged and silently stop reconciling — the realm import's own fail-open.
|
||||
raise SystemExit("FATAL: %s declares no webAuthnPolicyPasswordless* field — "
|
||||
"refusing to render an empty desired state" % IMPORT_FILE)
|
||||
if not policy.get(RP_ENTITY_NAME):
|
||||
raise SystemExit(
|
||||
"FATAL: %s does not declare a non-empty %s. Keycloak 26.0.6 applies the "
|
||||
"passwordless policy ONLY when that field is set, so every realm PUT would "
|
||||
"be accepted and change nothing" % (IMPORT_FILE, RP_ENTITY_NAME))
|
||||
|
||||
# ---- the flow tree, flattened per flow and ORDERED BY PRIORITY -----------------
|
||||
browser_flow = realm.get("browserFlow")
|
||||
if not isinstance(browser_flow, str) or not browser_flow:
|
||||
raise SystemExit("FATAL: %s declares no spec.realm.browserFlow — the flow could "
|
||||
"be perfect and still not be the one anyone logs in through"
|
||||
% IMPORT_FILE)
|
||||
raw_flows = realm.get("authenticationFlows") or []
|
||||
if not isinstance(raw_flows, list) or not raw_flows:
|
||||
raise SystemExit("FATAL: %s declares no authenticationFlows" % IMPORT_FILE)
|
||||
|
||||
flows: dict[str, object] = {}
|
||||
for fl in raw_flows:
|
||||
alias = fl.get("alias")
|
||||
if not isinstance(alias, str) or not alias:
|
||||
raise SystemExit("FATAL: %s has an authentication flow with no alias"
|
||||
% IMPORT_FILE)
|
||||
if alias in flows:
|
||||
raise SystemExit("FATAL: %s declares the flow %r twice"
|
||||
% (IMPORT_FILE, alias))
|
||||
execs = fl.get("authenticationExecutions") or []
|
||||
if not isinstance(execs, list) or not execs:
|
||||
raise SystemExit("FATAL: %s flow %r declares no authenticationExecutions"
|
||||
% (IMPORT_FILE, alias))
|
||||
ordered = []
|
||||
for ex in sorted(execs, key=lambda e: (e.get("priority") or 0)):
|
||||
is_flow = bool(ex.get("authenticatorFlow"))
|
||||
name = ex.get("flowAlias") if is_flow else ex.get("authenticator")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise SystemExit(
|
||||
"FATAL: %s flow %r has an execution with neither an authenticator "
|
||||
"nor a flowAlias" % (IMPORT_FILE, alias))
|
||||
requirement = ex.get("requirement")
|
||||
if not isinstance(requirement, str) or not requirement:
|
||||
raise SystemExit("FATAL: %s flow %r execution %r declares no requirement"
|
||||
% (IMPORT_FILE, alias, name))
|
||||
entry: dict[str, object] = {
|
||||
"kind": "flow" if is_flow else "authenticator",
|
||||
"name": name,
|
||||
"requirement": requirement,
|
||||
}
|
||||
cfg = ex.get("authenticatorConfig")
|
||||
if cfg is not None:
|
||||
if not isinstance(cfg, str) or not cfg:
|
||||
raise SystemExit("FATAL: %s flow %r execution %r has a non-string "
|
||||
"authenticatorConfig" % (IMPORT_FILE, alias, name))
|
||||
entry["configAlias"] = cfg
|
||||
ordered.append(entry)
|
||||
flows[alias] = {"topLevel": bool(fl.get("topLevel")), "executions": ordered}
|
||||
|
||||
if browser_flow not in flows:
|
||||
raise SystemExit("FATAL: %s binds browserFlow to %r but declares no such flow"
|
||||
% (IMPORT_FILE, browser_flow))
|
||||
# Every referenced subflow must be declared, or the reconciler's flattening walks
|
||||
# into a hole and the audit silently covers less than it claims.
|
||||
for alias, spec in flows.items():
|
||||
for ex in spec["executions"]:
|
||||
if ex["kind"] == "flow" and ex["name"] not in flows:
|
||||
raise SystemExit("FATAL: %s flow %r references subflow %r, which is not "
|
||||
"declared" % (IMPORT_FILE, alias, ex["name"]))
|
||||
|
||||
# ---- authenticator configs, copied verbatim ------------------------------------
|
||||
configs: dict[str, dict] = {}
|
||||
for cfg in realm.get("authenticatorConfig") or []:
|
||||
alias = cfg.get("alias")
|
||||
if not isinstance(alias, str) or not alias:
|
||||
raise SystemExit("FATAL: %s has an authenticatorConfig with no alias"
|
||||
% IMPORT_FILE)
|
||||
body = cfg.get("config") or {}
|
||||
if not isinstance(body, dict) or not body:
|
||||
raise SystemExit("FATAL: %s authenticatorConfig %r has no config body"
|
||||
% (IMPORT_FILE, alias))
|
||||
# Values are stringified because Keycloak stores authenticator config values as
|
||||
# strings. `negate: "false"` in the import and a stored "false" are the same
|
||||
# declaration, and a YAML-unquoted `false` must not become a Python bool that
|
||||
# then never matches.
|
||||
configs[alias] = {k: str(v) for k, v in sorted(body.items())}
|
||||
|
||||
# Every configAlias a flow references must be defined, else the reconciler reports
|
||||
# "git declares config X but defines no key/value for it" on every run forever.
|
||||
for alias, spec in flows.items():
|
||||
for ex in spec["executions"]:
|
||||
ca = ex.get("configAlias")
|
||||
if ca and ca not in configs:
|
||||
raise SystemExit(
|
||||
"FATAL: %s flow %r execution %r references authenticatorConfig %r, "
|
||||
"which is not declared" % (IMPORT_FILE, alias, ex["name"], ca))
|
||||
|
||||
# ---- the enrolment required action, DERIVED (see the header) -------------------
|
||||
offers_passkey = any(
|
||||
ex["kind"] == "authenticator" and ex["name"] == PASSKEY_AUTHENTICATOR
|
||||
for spec in flows.values() for ex in spec["executions"])
|
||||
if not offers_passkey:
|
||||
raise SystemExit(
|
||||
"FATAL: %s declares a webAuthnPolicyPasswordless* policy but no %r execution "
|
||||
"in any flow. The policy decides what a passkey must BE; without that "
|
||||
"execution nothing ever OFFERS one, so the policy is unreachable "
|
||||
"configuration and this desired state would be meaningless"
|
||||
% (IMPORT_FILE, PASSKEY_AUTHENTICATOR))
|
||||
|
||||
want = {
|
||||
"realm": realm_name,
|
||||
"webAuthnPolicyPasswordless": policy,
|
||||
# enabled: passkey enrolment must be possible, since the flow offers passkeys.
|
||||
# defaultAction FALSE: enrolment stays OPT-IN. True would force every user to
|
||||
# enrol a passkey at next login — the platform-wide lock-out the realm import's
|
||||
# "additive and optional" posture exists to avoid. Derived, not copied; see the
|
||||
# header for why the import cannot declare this itself.
|
||||
"requiredAction": {"alias": ENROL_ACTION, "enabled": True,
|
||||
"defaultAction": False},
|
||||
"browserFlow": browser_flow,
|
||||
"flows": flows,
|
||||
"authenticatorConfig": configs,
|
||||
}
|
||||
|
||||
body = json.dumps(want, indent=2, sort_keys=True, ensure_ascii=False)
|
||||
indented = "\n".join((" " + ln) if ln else "" for ln in body.splitlines())
|
||||
return (HEADER + BIND_BLOCK % (BIND_KEY, committed_bind(root))
|
||||
+ " %s: |\n" % DATA_KEY + indented + "\n")
|
||||
|
||||
|
||||
HEADER = """---
|
||||
# =============================================================================
|
||||
# GENERATED FILE — DO NOT EDIT BY HAND, except the one bind-browser-flow line below.
|
||||
#
|
||||
# source: platform/keycloak/keycloak-realm-import.yaml (spec.realm)
|
||||
# regenerate: python3 ci/render-keycloak-passkey-policy.py --write
|
||||
# gated by: conformance/tests/l0-static/keycloak-passkey-reconciler
|
||||
# (runs this renderer with --check on every merge)
|
||||
#
|
||||
# WHAT THIS IS: the DESIRED STATE for the `homelab` realm's PASSKEY configuration —
|
||||
# the WebAuthn passwordless policy, the enrolment required action, and the declared
|
||||
# browser authentication flow tree. platform/keycloak/passkey-reconciler.yaml mounts
|
||||
# it and (a) converges the policy and the required action onto it, (b) builds the
|
||||
# declared flow tree BLUE/GREEN under a new alias and verifies it by readback, never
|
||||
# writing the flow users are logging in through, and (c) BINDS that verified flow as
|
||||
# the realm's browserFlow ONLY when `bind-browser-flow` below is "true".
|
||||
#
|
||||
# EVERYTHING HERE IS GENERATED EXCEPT ONE LINE: `bind-browser-flow`. That is an
|
||||
# operator decision, carried through by the renderer from this committed file (see
|
||||
# ci/render-keycloak-passkey-policy.py). It defaults to "false".
|
||||
#
|
||||
# WHY IT IS A SEPARATE OBJECT: the realm import is a ONE-SHOT that never re-runs
|
||||
# once Done=True, so a realm-level field or a flow edited in git never reaches the
|
||||
# realm. Measured on prod 2026-10-18, hours after the passkey declaration merged:
|
||||
# Done=True / Started=False, with the whole passkey block inert. The reconciler is
|
||||
# the level-triggered loop that closes that; this file is what it reconciles
|
||||
# TOWARDS.
|
||||
#
|
||||
# WHY IT IS GENERATED AND NOT AUTHORED: a hand-written copy of the RP ID, the
|
||||
# user-verification requirement and the flow tree would be a second source of truth,
|
||||
# free to disagree with the realm import — the same "two declarations that can
|
||||
# disagree" shape as the bug being closed. It is rendered from the import and the
|
||||
# render is gated, so the two cannot drift.
|
||||
#
|
||||
# STRINGS ARE VERBATIM: webAuthnPolicyPasswordlessRpId is the ${BASE_DOMAIN}
|
||||
# substitution token exactly as the realm import writes it. Both files are rendered
|
||||
# by the SAME Flux Kustomization (./platform-config), so the token resolves to the
|
||||
# same value in both and the domain is decided in exactly one place.
|
||||
#
|
||||
# `requiredAction` is the ONE entry that is DERIVED rather than copied: the realm
|
||||
# import deliberately declares no `requiredActions` block, because Keycloak 26.0.6
|
||||
# treats that as REPLACE and declaring one action would delete CONFIGURE_TOTP and
|
||||
# nine more (a hard lock-out of every platform-admin). It is derived instead from
|
||||
# the presence of a webauthn-authenticator-passwordless execution in the flow:
|
||||
# passkeys are offered, therefore enrolment must be enabled — and defaultAction is
|
||||
# false, so enrolment stays opt-in.
|
||||
#
|
||||
# A field NOT declared on the realm in the import is simply ABSENT here, and the
|
||||
# reconciler leaves it alone. Absence means UNMANAGED, never "force to a default".
|
||||
# =============================================================================
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: %s
|
||||
namespace: %s
|
||||
labels:
|
||||
app.kubernetes.io/name: %s
|
||||
app.kubernetes.io/part-of: keycloak
|
||||
data:
|
||||
""" % (CONFIGMAP_NAME, NAMESPACE, CONFIGMAP_NAME)
|
||||
|
||||
BIND_BLOCK = """ # ---------------------------------------------------------------------------
|
||||
# THE BIND SWITCH -- the ONLY hand-edited line in this file. Exactly "true" or
|
||||
# "false", QUOTED.
|
||||
#
|
||||
# "false" the reconciler builds and verifies the candidate browser flow and
|
||||
# logs `WOULD BIND <alias>`, but NEVER changes the realm's browserFlow.
|
||||
# Whatever the realm is bound to -- including a rollback -- is left alone.
|
||||
# "true" it binds the verified candidate in one atomic PUT, and keeps the
|
||||
# realm bound to it: a binding changed by hand is changed back.
|
||||
#
|
||||
# GO LIVE: test the candidate in a browser first (platform/keycloak/README.md,
|
||||
# "Passkeys"), then a PR setting this to "true".
|
||||
# ROLL BACK: a PR setting this to "false" FIRST (or the next run rebinds), then
|
||||
# one write setting browserFlow back to the previous alias from the Job log.
|
||||
# ---------------------------------------------------------------------------
|
||||
%s: "%s"
|
||||
"""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
g = ap.add_mutually_exclusive_group()
|
||||
g.add_argument("--write", action="store_true", help="write %s" % OUT_FILE)
|
||||
g.add_argument("--check", action="store_true",
|
||||
help="exit 1 if the committed file has drifted from the source")
|
||||
args = ap.parse_args()
|
||||
|
||||
want = render(ROOT)
|
||||
target = ROOT / OUT_FILE
|
||||
|
||||
if args.write:
|
||||
target.write_text(want, encoding="utf-8")
|
||||
print("wrote %s" % OUT_FILE)
|
||||
return 0
|
||||
|
||||
if args.check:
|
||||
if not target.is_file():
|
||||
print("CHECK FAIL: %s does not exist. Run: python3 "
|
||||
"ci/render-keycloak-passkey-policy.py --write" % OUT_FILE,
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
have = target.read_text(encoding="utf-8")
|
||||
if have == want:
|
||||
print("passkey-desired-state.yaml is in sync with %s" % IMPORT_FILE)
|
||||
return 0
|
||||
print("CHECK FAIL: %s has DRIFTED from %s — the reconciler would converge the "
|
||||
"realm on a passkey policy the realm import does not declare, and build "
|
||||
"the login flow from a tree nobody authored.\n"
|
||||
"Regenerate with: python3 ci/render-keycloak-passkey-policy.py --write\n"
|
||||
% (OUT_FILE, IMPORT_FILE), file=sys.stderr)
|
||||
for line in difflib.unified_diff(have.splitlines(True), want.splitlines(True),
|
||||
fromfile="committed " + OUT_FILE,
|
||||
tofile="rendered from " + IMPORT_FILE):
|
||||
sys.stderr.write(line)
|
||||
return 1
|
||||
|
||||
sys.stdout.write(want)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
# Flux Kustomization -> ./platform/buildkit — rootless BuildKit, the image builder CI uses
|
||||
# (.gitea/workflows/images.yaml). Its own Kustomization so a stuck Certificate or a failing
|
||||
# builder can never stall any other tier (Flux health-checks Certificates under wait:true).
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: buildkit
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: platform # cert-manager CRDs (Issuer/Certificate) come from its HelmRelease
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/buildkit
|
||||
@@ -0,0 +1,61 @@
|
||||
---
|
||||
# Flux Kustomization: Cloudflare Tunnel connector — reconciles ./platform/cloudflared.
|
||||
#
|
||||
# ISOLATION IS THE POINT OF THIS FILE. The Deployment it applies CANNOT become Ready until the
|
||||
# operator enrols the tunnel (runbooks/cloudflare-tunnel.md): it ships with REPLACE-ME credentials
|
||||
# and a placeholder UUID, and a wrong or revoked credential later produces the same state. So:
|
||||
#
|
||||
# * ITS OWN Kustomization, not a line in platform/kustomization.yaml. `platform` is wait:true and
|
||||
# gates platform-config, which gates every apps wave; an unenrolled tunnel there would be a
|
||||
# platform-wide stall ON MERGE, with the cause (a missing manual step) invisible from any Flux
|
||||
# condition. Same reasoning, and same failure shape, as vpn-ingress's unfilled server key.
|
||||
# * wait: false. Its pods are legitimately not-Ready whenever the credentials are absent or the
|
||||
# edge is unreachable. Health is reported by the CloudflaredTunnelDown / ReplicasDegraded
|
||||
# alerts (platform/cloudflared/alerts.yaml), not by this Kustomization's Ready condition —
|
||||
# which reaches Ready on APPLY success.
|
||||
# * NOTHING dependsOn THIS, and nothing should. A missing or invalid credentials Secret can then
|
||||
# only ever affect the tunnel itself. conformance/tests/l0-static/cloudflared-ingress-rules
|
||||
# fails the gate if any Kustomization gains `dependsOn: cloudflared`, or if this one stops
|
||||
# being wait:false.
|
||||
# * No Certificate, no HelmRelease, no CRD: nothing in the path for Flux's health checks to wait
|
||||
# on even if wait were flipped (the Certificate-stall lesson).
|
||||
#
|
||||
# dependsOn: [platform-config] — A REAL EDGE.
|
||||
# 1. CRD ORDERING: PodMonitor and PrometheusRule are monitoring.coreos.com CRs whose CRDs are
|
||||
# installed by kube-prometheus-stack in `platform`; platform-config dependsOn platform, so this
|
||||
# edge carries that guarantee transitively (Flux aborts a whole pass on an unknown kind).
|
||||
# 2. THE FAR SIDE OF THE TUNNEL: the Gateway (and so the cilium-gateway-shared Service the ingress
|
||||
# rules point at) is applied by platform-config. A tunnel that comes up before its origin is a
|
||||
# support call.
|
||||
#
|
||||
# ZERO EFFECT ON LIVE TRAFFIC UNTIL DNS MOVES. Cloudflare only sends a hostname through a tunnel
|
||||
# when that hostname's DNS record is a CNAME to <uuid>.cfargotunnel.com. This change creates no
|
||||
# such record and leaves external-dns, the Gateway annotations, Pi-hole and CoreDNS untouched.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/cloudflared
|
||||
# REQUIRED, not boilerplate: platform/cloudflared/config.yaml renders `${BASE_DOMAIN}` into every
|
||||
# ingress hostname, and alerts.yaml into both runbook_url annotations. Without this block the
|
||||
# token becomes the EMPTY STRING and the Kustomization still reports Ready (substitution fails
|
||||
# OPEN), leaving ingress rules for `plex.` and `*.` — a tunnel that 404s everything at best.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption is layered per-environment: clusters/dev and clusters/prod each patch THIS
|
||||
# Kustomization by name with a `decryption:` stanza for cloudflared-credentials.sops.yaml.
|
||||
# Without it the Secret applies cleanly full of ENC[...] and cloudflared cannot authenticate.
|
||||
@@ -102,6 +102,7 @@ data:
|
||||
# S3_BACKUP_ENDPOINT, CNPG_BACKUP_PATH; pihole + pihole-dns: PIHOLE_DNS_VIP, LAN_CIDR, plus
|
||||
# GATEWAY_VIP/UNIFI_L4_VIP/CRAFTY_L4_VIP read BY pihole to author its LAN split-horizon answers;
|
||||
# plex-remote + pihole: PLEX_L4_VIP),
|
||||
# vpn-ingress (WG_CLIENT_CIDR),
|
||||
# velero (S3_BACKUP_ENDPOINT, S3_BACKUP_REGION,
|
||||
# VELERO_BACKUP_BUCKET, VELERO_BACKUP_PREFIX), apps-vms (UNIFI_L4_VIP). A ${TOKEN} in an
|
||||
# UNSUBSTITUTED path would ship the literal string into the cluster.
|
||||
@@ -207,27 +208,54 @@ data:
|
||||
# prod -> 192.168.2.0/24 (prod LAN; nodes .10/.11/.12)
|
||||
LAN_CIDR: "192.168.1.0/24" # ingress scope for the PiHole DNS listener
|
||||
|
||||
# --- Remote-access VPN client subnet (platform/vpn-ingress, WireGuard) --------------------------
|
||||
# The tunnel subnet handed to REMOTE VPN clients. `.1` is the server (the hostNetwork DaemonSet,
|
||||
# on every node); `.2`-`.254` are devices, allocated one /32 each in
|
||||
# platform/vpn-ingress/peers.yaml. Consumed twice in the pod's config-assembly script: to derive
|
||||
# the server's own address, and as the `-s` match of the MASQUERADE rule that makes LAN replies
|
||||
# routable.
|
||||
#
|
||||
# IT IS A KNOB RATHER THAN A LITERAL FOR ONE REASON: it is the only address range in this cluster
|
||||
# that must be checked against EVERY OTHER range at once, so it belongs where those are declared.
|
||||
# 10.90.0.0/24 was chosen to collide with none of them:
|
||||
# 10.0.0.0/16 Cilium cluster-pool pod CIDR (${CLUSTER_POD_CIDR})
|
||||
# 10.43.0.0/16 k3s service CIDR (measured live 2026-10-05: kubernetes svc
|
||||
# clusterIP 10.43.0.1)
|
||||
# 10.72.192.0/18 the PIA tunnel's pushed addresses (platform/vpn-egress)
|
||||
# 192.168.1.0/24 dev LAN / pfSense LAN (${LAN_CIDR} base default)
|
||||
# 192.168.2.0/24 prod LAN (${LAN_CIDR} on prod)
|
||||
#
|
||||
# NO PER-ENVIRONMENT OVERRIDE, and that is deliberate rather than an omission: unlike every VIP
|
||||
# above, this range is NOT drawn from the LAN. It is private to the tunnel, so the same value is
|
||||
# correct in both environments and a prod patch would be a no-op that implies otherwise.
|
||||
#
|
||||
# IF YOU CHANGE IT, every already-issued client profile breaks (its `Address` is from this range)
|
||||
# and must be re-rendered — so treat it as fixed once the first device is enrolled.
|
||||
WG_CLIENT_CIDR: "10.90.0.0/24" # remote VPN clients; .1 = server. NOT a LAN range
|
||||
|
||||
# --- Bulk (external NFS) storage host (docs/04 bulk tier) ---------------------------------------
|
||||
# THE address of the Proxmox host whose ZFS datasets back every `class: bulk` volume: the media
|
||||
# library (Storage/plex, 8.33 T — plex, sonarr, radarr) and the legacy Nextcloud archive
|
||||
# THE address of the NAS whose ZFS datasets back every `class: bulk` volume: the media library
|
||||
# (Storage/plex, 8.33 T — plex, sonarr, radarr, qbittorrent) and the legacy Nextcloud archive
|
||||
# (Storage/nextcloud, 270 G). The renderer emits it into each static NFS PersistentVolume's
|
||||
# `spec.nfs.server` (catalog/render.py, BULK_NFS_SERVER); the export PATH is per-dataset and is
|
||||
# declared in the ServiceDefinition, because the path is identical in every environment and only
|
||||
# the ADDRESS is not.
|
||||
#
|
||||
# WHY THIS IS A KNOB AND NOT A LITERAL — the two clusters are on OPPOSITE SIDES of pfSense.
|
||||
# The storage host has exactly one interface, vmbr0 192.168.1.9/24, on pfSense's LAN.
|
||||
# dev nodes are 192.168.1.40/.41/.42 — the SAME L2 segment. They reach it directly, and one
|
||||
# NFS export to them already exists on that host (/mnt/Storage/k8s-volumes in
|
||||
# /etc/exports), so this path is proven, not theoretical. Base default = the dev value,
|
||||
# per the convention of the LAN block above.
|
||||
# prod nodes are 192.168.2.10/.11/.12 — pfSense's WAN. tcp/2049 does NOT traverse that today;
|
||||
# 22 and 8006 answer only because pfSense port-forwards them via 192.168.2.102. Opening
|
||||
# the storage path is a PENDING OPERATOR DECISION with two candidate outcomes that give
|
||||
# DIFFERENT values here: 192.168.2.102 (another port-forward) or 192.168.1.9 (a route).
|
||||
# So prod holds a loud TBD-PROD-* placeholder — see clusters/prod/kustomization.yaml for the
|
||||
# blast radius of shipping it unset, which is NOT the usual "fails to apply".
|
||||
BULK_NFS_SERVER: "192.168.1.9"
|
||||
# IT IS NO LONGER A CROSS-NETWORK KNOB (2026-10-07). This used to read "the two clusters are on
|
||||
# OPPOSITE SIDES of pfSense": the datasets lived on a Proxmox host whose only interface was
|
||||
# vmbr0 192.168.1.9/24 on pfSense's LAN, which prod (192.168.2.0/24, pfSense's WAN) could reach
|
||||
# only through a DNAT at 192.168.2.102. Both pools were physically moved onto a standalone
|
||||
# TrueNAS at 192.168.2.30 — prod's OWN /24 — so there is no crossing to arrange and no candidate
|
||||
# values to choose between. Measured 2026-10-07: 192.168.2.102:2049 no longer answers;
|
||||
# 192.168.2.30:2049 and :111 do. There is now exactly ONE storage host and one correct value;
|
||||
# this stays a knob only because the renderer reads it per-environment.
|
||||
#
|
||||
# *** CHANGING THIS VALUE IS NOT A PLAIN RECONCILE. *** A bound PersistentVolume's spec.nfs is
|
||||
# immutable — measured by server dry-run 2026-10-07:
|
||||
# spec.persistentvolumesource: Forbidden: spec.persistentvolumesource is immutable after creation
|
||||
# so Flux's apply FAILS rather than updating, and the pods keep their old mount. The PVs must be
|
||||
# deleted and recreated; the operator sequence is docs/BULK-NFS-STORAGE.md §10.
|
||||
BULK_NFS_SERVER: "192.168.2.30"
|
||||
|
||||
# Off-cluster S3 backup target (R-STATE-03, R-IDP-05) — the on-Proxmox MinIO, plain HTTP, path-style.
|
||||
# Consumed by BOTH backup paths: every CNPG Cluster's barmanObjectStore.endpointURL (keycloak-db,
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
---
|
||||
# Flux Kustomization: kured (Kubernetes Reboot Daemon) — reconciles ./platform/kured.
|
||||
# Performs the cordon / drain / reboot / uncordon of a node that has a pending OS reboot, strictly
|
||||
# one node at a time, inside a nightly window, gated on Longhorn volume health. Design rationale,
|
||||
# every tuned value, and the honest list of what is NOT mitigated: platform/kured/helmrelease.yaml.
|
||||
#
|
||||
# WHY ITS OWN KUSTOMIZATION, AND NOT ONE LINE IN platform/kustomization.yaml
|
||||
#
|
||||
# The same argument the Cilium split-out recorded in platform/kustomization.yaml and intel-gpu-plugin
|
||||
# reused, except the consequence is worse here than for either of them:
|
||||
#
|
||||
# * `platform` is `wait: true` over twelve-plus components, so its Ready condition cannot tell you
|
||||
# "the reboot daemon is wedged" from "longhorn is slow".
|
||||
# * The only emergency valve would be `flux suspend kustomization platform` — freezing storage,
|
||||
# TLS, DNS, ingress, monitoring, Keycloak and KubeVirt in order to stop touching a DaemonSet.
|
||||
# * That tier's 10m interval / 5m timeout are called out there as WRONG for a 3-node DaemonSet
|
||||
# rollout. This is a 3-node DaemonSet rollout.
|
||||
# * And `platform` gates platform-config, which gates apps-wave-0 (wait:true), which gates
|
||||
# apps-wave-1. A reboot daemon that failed to go healthy inside that tier would stall EVERY
|
||||
# APPLICATION in the cluster on "dependency not ready". A component added to make maintenance
|
||||
# safer must not acquire the ability to cause an outage; that trade is the whole argument.
|
||||
#
|
||||
# dependsOn: [platform] — AND THIS ONE IS A REAL EDGE, NOT CONVENTION
|
||||
#
|
||||
# Not `cilium` (what intel-gpu-plugin uses) and not `platform-config` (what the leaf UIs use). It is
|
||||
# `platform`, for two independent reasons:
|
||||
#
|
||||
# 1. CRD ORDERING (F-MON1). platform/kured ships a PrometheusRule, and the chart generates a
|
||||
# ServiceMonitor. Both are CRs whose CRDs (monitoring.coreos.com) are installed by
|
||||
# kube-prometheus-stack INSIDE the `platform` tier. Flux dry-runs every object in a pass before
|
||||
# applying any of them, so a pass containing an unknown kind ABORTS — the failure that made the
|
||||
# CDI StorageProfiles a true deadlock on prod on 2026-09-05 (see
|
||||
# clusters/base/platform-config-cdi.yaml). Depending on `platform` guarantees the CRDs exist
|
||||
# before this pass is attempted.
|
||||
# 2. FUNCTIONAL PREREQUISITE. kured's reboot gate QUERIES Prometheus
|
||||
# (`prometheusUrl` in the HelmRelease). Prometheus lives in platform/monitoring. If kured ran
|
||||
# first it would get connection errors, treat the gate as unevaluable and skip every reboot —
|
||||
# the correct fail-safe direction, but silent. Longhorn, whose volume health IS the gate, is
|
||||
# also in that tier.
|
||||
#
|
||||
# It does NOT need platform-config: no Keycloak, no TLS, no split-horizon DNS, no Gateway, no
|
||||
# HTTPRoute. kured serves no HTTP to users.
|
||||
#
|
||||
# wait: true — A HEALTH SIGNAL, NOT A GATE. Following intel-gpu-plugin rather than the leaf-UI
|
||||
# template. A leaf UI is legitimately not-Ready for long stretches of a cold start while it waits on
|
||||
# Keycloak and DNS, so its health is uninformative; kured has no such dependency — it is a DaemonSet
|
||||
# that opens a socket and watches a file, so on healthy nodes it is Ready in seconds and not-Ready
|
||||
# genuinely means BROKEN. This turns `flux get kustomization kured` into a one-line answer to "is
|
||||
# reboot automation live on all three nodes", which is the question the operator must be able to
|
||||
# answer before trusting a kernel upgrade to it.
|
||||
#
|
||||
# NOTHING IN THE GRAPH dependsOn THIS, so wait:true gates nothing — the blast radius of an unhealthy
|
||||
# kured is this Kustomization's own Ready condition. Do not add `dependsOn: [kured]` anywhere: that
|
||||
# would hand a node-rebooting DaemonSet the power to stall whatever depends on it, which is precisely
|
||||
# what the placement above exists to prevent.
|
||||
#
|
||||
# prune: true is safe and deliberate. If this Kustomization is deleted, kured's DaemonSet goes with
|
||||
# it and reboots simply stop happening — nodes are left uncordoned and running. The one caveat is
|
||||
# that pruning mid-reboot would leave a node CORDONED with no daemon to uncordon it; that is a
|
||||
# `kubectl uncordon` away and is covered in runbooks/node-reboot-automation.md.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: kured
|
||||
namespace: flux-system
|
||||
spec:
|
||||
# A HelmRelease plus five static manifests, nothing to re-render; kustomize-controller is
|
||||
# source-triggered, so this is only the drift-correction floor.
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
# 10m, not the 5m the `platform` tier uses: a 3-node DaemonSet rollout under `wait: true` can
|
||||
# exceed 5m on a cold start (image pull on three nodes), which is one of the reasons the Cilium
|
||||
# split-out gave for leaving that tier.
|
||||
timeout: 10m
|
||||
prune: true
|
||||
# See the header: a health SIGNAL for a component nothing depends on, not a gate.
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: platform # monitoring CRDs + Prometheus + Longhorn; see the header, this is load-bearing
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/kured
|
||||
# REQUIRED, not boilerplate. platform/kured/alerts.yaml renders `${BASE_DOMAIN}` into the
|
||||
# `runbook_url` annotation of all three rules. Without this block that token becomes the EMPTY
|
||||
# STRING and the Kustomization still reports Ready — Flux's substitution fails OPEN — so every
|
||||
# alert would ship a dead link to `https:///runbooks/node-reboot-automation`. The gate stage
|
||||
# `flux-postbuild-substitution` asserts the converse (a rendered token implies this block), and
|
||||
# BASE_DOMAIN is declared in clusters/base/cluster-vars.yaml.
|
||||
#
|
||||
# Nothing else in this path contains `${...}` or a bare `$`: the HelmRelease's alertFilterRegexp is
|
||||
# a plain anchored alternation, and there are no shell snippets anywhere in the component — which
|
||||
# is deliberate, because envsubst would eat a braced shell variable and `$$` would trip the
|
||||
# dollar-collapse assertion in the same check.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
@@ -71,4 +71,43 @@ resources:
|
||||
# controller must NEVER gate the platform chain; same non-gating rationale as headlamp/oauth2-proxy.
|
||||
# Carries velero-cloud.sops.yaml, so clusters/dev adds a SOPS decryption patch for it (by name).
|
||||
- backup.yaml # Flux Kustomization -> ./platform/velero (dependsOn platform-config)
|
||||
# WireGuard remote-access VPN (north-south INGRESS, udp/51820) — POST-platform tier
|
||||
# (dependsOn platform-config, wait:false). NOT the same thing as platform/vpn-egress, which sends
|
||||
# pods OUT through PIA; this lets remote CLIENTS IN. Non-gating is load-bearing rather than
|
||||
# conventional here: its DaemonSet cannot start until the operator creates the out-of-band
|
||||
# `wg-server-key` Secret, so in the gating `platform` tier an unfinished manual step would have
|
||||
# stalled every apps wave. Carries no SOPS secret, so neither overlay adds a decryption patch.
|
||||
- vpn-ingress.yaml # Flux Kustomization -> ./platform/vpn-ingress (dependsOn platform-config)
|
||||
# CVE visibility scanner (docs/17, ADR-0027) — POST-platform tier (dependsOn platform-config,
|
||||
# wait:false): a failing scanner must never gate the platform, and nothing dependsOn it. Carries the
|
||||
# oauth2-proxy credential SOPS file, so both overlays add a decryption patch by name.
|
||||
- vuln-scanner.yaml # Flux Kustomization -> ./platform/vuln-scanner (dependsOn platform-config)
|
||||
# Cloudflare Tunnel connector (north-south INGRESS from the Cloudflare edge) — POST-platform tier
|
||||
# (dependsOn platform-config, wait:false), and NOTHING dependsOn it. Non-gating is load-bearing,
|
||||
# as for vpn-ingress: the Deployment cannot become Ready until the operator enrols the tunnel
|
||||
# (runbooks/cloudflare-tunnel.md), so in a gating tier an unfinished manual step — or a revoked
|
||||
# credential later — would stall every apps wave. Carries cloudflared-credentials.sops.yaml, so
|
||||
# both overlays add a SOPS decryption patch for it (by name). See clusters/base/cloudflared.yaml.
|
||||
- cloudflared.yaml # Flux Kustomization -> ./platform/cloudflared (dependsOn platform-config)
|
||||
# PM agent — reads Gitea issues and runs one worker agent per assignment that authors a
|
||||
# change, gates it and opens a pull request. POST-APP tier (dependsOn apps-wave-0,
|
||||
# wait:false): what it needs is GITEA, not Keycloak or the Gateway, and it must never gate
|
||||
# anything — it CrashLoops by design while its credentials are still placeholders. Carries
|
||||
# pm-agent-credentials.sops.yaml, so both overlays add a SOPS decryption patch for it (by
|
||||
# name). See clusters/base/pm-agent.yaml and platform/pm-agent/README.md.
|
||||
- pm-agent.yaml # Flux Kustomization -> ./platform/pm-agent (dependsOn apps-wave-0)
|
||||
# kured (Kubernetes Reboot Daemon) — its OWN Kustomization (dependsOn platform, wait:true), NOT a
|
||||
# line in platform/kustomization.yaml. Same reasoning as the Cilium and intel-gpu-plugin
|
||||
# split-outs, with a sharper consequence: `platform` gates platform-config and every apps wave, so
|
||||
# a wedged reboot DaemonSet placed in that wait:true tier could stall every application in the
|
||||
# cluster — a component added to make maintenance SAFER must not be able to cause an outage.
|
||||
# wait:true here is a HEALTH SIGNAL, not a gate: nothing dependsOn it, and nothing should.
|
||||
# The `platform` edge is load-bearing rather than conventional — it supplies the
|
||||
# monitoring.coreos.com CRDs this component's PrometheusRule/ServiceMonitor need (F-MON1), and
|
||||
# Prometheus itself, which kured QUERIES before every reboot to gate on Longhorn volume health.
|
||||
# It reboots a node only when /var/run/reboot-required exists, which on these nodes only a
|
||||
# deliberate operator kernel upgrade creates — the human gate is the sentinel, not the daemon.
|
||||
# See clusters/base/kured.yaml and platform/kured/helmrelease.yaml.
|
||||
- kured.yaml # Flux Kustomization -> ./platform/kured (dependsOn platform)
|
||||
- buildkit.yaml # Flux Kustomization -> ./platform/buildkit (dependsOn platform)
|
||||
- cluster-vars.yaml # overlay knob surface (base defaults)
|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
# Flux Kustomization: the PM agent service — reconciles ./platform/pm-agent.
|
||||
#
|
||||
# POST-APP TIER (dependsOn apps-wave-0). What this service actually depends on is GITEA: the
|
||||
# dispatcher authenticates to it at startup, reads issue threads from it, and every worker
|
||||
# clones and pushes to it. Gitea is in apps-wave-0, so that is the dependency — not
|
||||
# platform-config, which brings up the Keycloak realm, the wildcard certificate, the Gateway
|
||||
# and the L2 pool. This service has no UI, serves no traffic, takes no ingress and holds no
|
||||
# certificate, so none of those are prerequisites and claiming them would be a lie that
|
||||
# delays its reconcile behind four things it does not use.
|
||||
#
|
||||
# NON-GATING (wait:false), and here that is not merely prudent — the pod is EXPECTED to be
|
||||
# unhealthy for a while:
|
||||
# * the committed credentials are PLACEHOLDERS, so until the operator replaces them the
|
||||
# dispatcher fails its Gitea authentication at startup and CrashLoops, by design
|
||||
# (platform/pm-agent/pm-agent-credentials.sops.yaml.example);
|
||||
# * the image digest is likewise a placeholder until the operator pushes the image, so
|
||||
# before that the pod cannot even pull.
|
||||
# Either of those inside a wait:true tier would stall everything downstream of it. wait:false
|
||||
# guarantees this service's health can NEVER gate anything, even if some future resource is
|
||||
# mistakenly pointed dependsOn it. Same non-gating leaf rationale as
|
||||
# headlamp/homepage/crafty/backup — with a stronger reason than any of them.
|
||||
#
|
||||
# PRUNE: true. The whole namespace is disposable — the dispatcher keeps no durable state (the
|
||||
# Gitea issue thread is its state) and worker Jobs are ephemeral by construction. So removing
|
||||
# this service from git really should remove it from the cluster, with nothing to orphan.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: pm-agent
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: apps-wave-0 # Gitea: the repository this service reads issues from and pushes to
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/pm-agent
|
||||
# cluster-vars is substituted for parity with every other tier. NOTE for anyone editing
|
||||
# platform/pm-agent/: envsubst runs over the WHOLE rendered manifest, comments included, so
|
||||
# a `${...}` meant for a shell would be replaced with the empty string before the container
|
||||
# starts (the hazard catalog/README.md documents at length — it silently killed gitea's and
|
||||
# nextcloud's OIDC registration for days). Nothing under platform/pm-agent/ uses a `${}`
|
||||
# token today, and the two scripts that DO contain shell live in the image, not in a
|
||||
# manifest, which is one more reason they are baked in rather than mounted from a ConfigMap.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption for pm-agent-credentials.sops.yaml is layered per-environment by the
|
||||
# overlay (clusters/prod and clusters/dev patch THIS Kustomization by name), identical to
|
||||
# crafty/headlamp/oauth2-proxy/backup.
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
# Flux Kustomization: WireGuard remote-access VPN — reconciles ./platform/vpn-ingress.
|
||||
#
|
||||
# POST-PLATFORM TIER (dependsOn platform-config), NON-GATING (wait:false). Not a line in
|
||||
# platform/kustomization.yaml, for the standard reason plus one specific to this component:
|
||||
#
|
||||
# * STANDARD: a leaf must never sit in the gating `platform` tier. While `platform` is not-Ready
|
||||
# NOTHING downstream reconciles — the incident behind the headlamp/oauth2-proxy/homepage moves
|
||||
# (DECISIONS-LOG: "Headlamp gating stall").
|
||||
# * SPECIFIC, AND THE REASON THIS IS NOT A JUDGEMENT CALL: this DaemonSet cannot start until the
|
||||
# operator replaces the `REPLACE-ME` placeholder in platform/vpn-ingress/wg-server-key.sops.yaml
|
||||
# with a real key (runbooks/vpn-remote-access.md step 2). That is the correct fail-closed state — an unenrolled VPN must not come up — but in
|
||||
# the gating tier it would have been a self-inflicted platform-wide outage ON MERGE, with the
|
||||
# cause (a missing manual step) invisible from any Flux condition.
|
||||
#
|
||||
# WHY dependsOn platform-config AND NOT cilium. This component needs no Gateway, no TLS cert, no
|
||||
# Keycloak and no LB-IPAM pool — it has no HTTPRoute and no LoadBalancer Service, and hostNetwork
|
||||
# means it does not even need a Cilium endpoint. It could legally reconcile much earlier. It is
|
||||
# pinned behind platform-config anyway so that the ONE thing remote clients actually consume
|
||||
# through it — split-horizon DNS and the shared Gateway — exists by the time the tunnel does.
|
||||
# A tunnel that comes up before the things on the far side of it is a support call, not a feature.
|
||||
#
|
||||
# wait:false: nothing in the graph dependsOn this, and its pods are legitimately not-Ready whenever
|
||||
# the server key is absent or the `wireguard` kernel module has not been loaded on a node. Neither
|
||||
# condition should be able to gate anything else, even if a future resource is mistakenly pointed
|
||||
# at it.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vpn-ingress
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/vpn-ingress
|
||||
# REQUIRED, not boilerplate: platform/vpn-ingress/daemonset.yaml renders ${WG_CLIENT_CIDR} into
|
||||
# the config-assembly script. Without this block that token becomes the EMPTY STRING and the
|
||||
# Kustomization still reports Ready (Flux's substitution fails OPEN — ci:
|
||||
# flux-postbuild-substitution), leaving an initContainer that derives a server address of ".1/"
|
||||
# and a MASQUERADE rule with no source. Both fail loudly at pod start, but the Kustomization
|
||||
# would look healthy.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption is layered per-environment by the overlay (clusters/dev and clusters/prod each
|
||||
# patch THIS Kustomization by name), identical to headlamp/crafty/velero — this path carries
|
||||
# platform/vpn-ingress/wg-server-key.sops.yaml, the WireGuard server private key. The `sops-age`
|
||||
# Secret is created out of band and referenced by NAME only. Without the overlay patch the Secret
|
||||
# applies cleanly full of ENC[...] and only the tunnel is broken, so the patch is as load-bearing
|
||||
# as the encrypted file itself.
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
# Flux Kustomization: the CVE visibility scanner (docs/17, ADR-0027) — reconciles ./platform/vuln-scanner.
|
||||
#
|
||||
# ISOLATION IS THE POINT, as for clusters/base/cloudflared.yaml. A failing scanner, an unbound PVC or a
|
||||
# pending Certificate must never become a stalled platform, so:
|
||||
# * ITS OWN Kustomization, not a line in platform/kustomization.yaml;
|
||||
# * wait: false — its health is reported by the VulnScanStale/VulnDatabaseStale/VulnCoverageGap/
|
||||
# VulnCriticalFixAvailable/VulnTriageBotSilent alerts (platform/vuln-scanner/alerts.yaml), not by
|
||||
# this object's Ready condition, which reaches Ready on apply success;
|
||||
# * NOTHING dependsOn THIS. conformance/tests/l0-static/vuln-scanner-isolation fails the gate if any
|
||||
# Kustomization gains `dependsOn: vuln-scanner` or this one stops being wait:false.
|
||||
#
|
||||
# dependsOn: [platform-config] — a real edge: PrometheusRule/ServiceMonitor/Gateway API CRDs, the
|
||||
# Longhorn StorageClass, the Gateway, Keycloak's realm, and Longhorn itself come up through
|
||||
# `platform`/`platform-config` first.
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vuln-scanner
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 2m
|
||||
timeout: 5m
|
||||
prune: true
|
||||
wait: false
|
||||
dependsOn:
|
||||
- name: platform-config
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./platform/vuln-scanner
|
||||
# REQUIRED: the HTTPRoute hostname, the oauth2-proxy issuer/redirect/cookie-domain args and every
|
||||
# alert's runbook_url are written `${BASE_DOMAIN}`. Substitution fails OPEN, so the file
|
||||
# conformance/tests/l0-static/flux-postbuild-substitution audits this tier.
|
||||
postBuild:
|
||||
substituteFrom:
|
||||
- kind: ConfigMap
|
||||
name: cluster-vars
|
||||
# SOPS decryption is layered per-environment: clusters/dev and clusters/prod each patch THIS
|
||||
# Kustomization by name with a `decryption:` stanza for the oauth2-proxy credentials.
|
||||
@@ -122,6 +122,75 @@ patches:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# vpn-ingress (the WireGuard remote-access VPN, clusters/base/vpn-ingress.yaml) carries
|
||||
# platform/vpn-ingress/wg-server-key.sops.yaml — the WireGuard SERVER private key, the one
|
||||
# secret in that component. Without this stanza kustomize hands Flux the raw ciphertext and the
|
||||
# Secret applies full of ENC[...]: the APPLY SUCCEEDS and every Flux condition stays green, and
|
||||
# the only symptom is a VPN that no client can handshake with. The DaemonSet's init container
|
||||
# validates the value with `wg pubkey` specifically so that state is a named CrashLoop instead.
|
||||
# The `sops-age` Secret is out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vpn-ingress
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# vuln-scanner (the CVE visibility scanner, clusters/base/vuln-scanner.yaml) carries
|
||||
# platform/vuln-scanner/vuln-scanner-oauth2-proxy-credentials.sops.yaml — the per-pod oauth2-proxy
|
||||
# client and cookie secrets. Without this stanza the Secret applies full of ENC[...] and the
|
||||
# sidecar cannot authenticate; the rest of the scanner runs regardless. `sops-age` is out-of-band.
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vuln-scanner
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# cloudflared (the Cloudflare Tunnel connector, clusters/base/cloudflared.yaml) carries
|
||||
# platform/cloudflared/cloudflared-credentials.sops.yaml — the tunnel credentials JSON, the one
|
||||
# secret in that component. Without this stanza the Secret applies full of ENC[...]: the APPLY
|
||||
# SUCCEEDS, the Kustomization stays Ready, and the only symptom is a connector that cannot
|
||||
# authenticate to the edge (CloudflaredTunnelDown fires). The `sops-age` Secret is out-of-band
|
||||
# (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# pm-agent (the Program Manager agent service, clusters/base/pm-agent.yaml) carries
|
||||
# pm-agent-credentials.sops.yaml — the Gitea write token for the dedicated non-admin bot
|
||||
# account and the Anthropic subscription token from `claude setup-token`. Without this
|
||||
# stanza kustomize hands Flux the raw SOPS document and the Secret applies full of
|
||||
# ciphertext: the APPLY SUCCEEDS, and the only symptom is a dispatcher that cannot
|
||||
# authenticate to Gitea — so this stanza is as load-bearing as the file it decrypts (same
|
||||
# silent-failure shape as the apps-vms note below/above). The `sops-age` Secret is created
|
||||
# out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: pm-agent
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# apps-vms (the KubeVirt VM tier, clusters/base/apps-vms.yaml) gained its FIRST encrypted Secret
|
||||
# on 2026-09-14: apps/vms/unifi/unifi-uos-installer.sops.yaml, the operator's UOS installer URL,
|
||||
# which 30-virtualmachine.yaml mounts into the guest as a disk. Without this stanza kustomize
|
||||
@@ -242,15 +311,15 @@ patches:
|
||||
- op: replace
|
||||
path: /data/CRAFTY_L4_VIP
|
||||
value: "192.168.1.55"
|
||||
# Bulk (external NFS) storage host: the Proxmox host itself, on the SAME LAN as dev's nodes
|
||||
# (192.168.1.40/.41/.42), which already hold an NFS export from it (/mnt/Storage/k8s-volumes).
|
||||
# Restated verbatim rather than inherited, per this overlay's "supply every knob" convention.
|
||||
# Dev is therefore the environment in which the bulk manifests can actually be VALIDATED
|
||||
# end-to-end before prod's network path exists — provided the additive `zfs set sharenfs=`
|
||||
# change described in docs/BULK-NFS-STORAGE.md is applied for the dev node addresses.
|
||||
# Bulk (external NFS) storage host. THIS VALUE IS INERT — dev is decommissioned, its nodes
|
||||
# (192.168.1.40/.41/.42) no longer exist, and the Proxmox host this used to name (192.168.1.9)
|
||||
# went with the pools it held. It is restated per this overlay's "supply every knob"
|
||||
# convention, not because anything consumes it. It carries the real address so that no file
|
||||
# in the tree renders a host that is gone; it has NOT been proven from a dev node, because
|
||||
# there is no dev node to prove it from.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
value: "192.168.1.9"
|
||||
value: "192.168.2.30"
|
||||
# Off-cluster backup target: the on-Proxmox MinIO (plain HTTP, path-style) that every CNPG
|
||||
# barmanObjectStore and Velero's backupStorageLocation write to.
|
||||
#
|
||||
|
||||
@@ -113,6 +113,75 @@ patches:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# vpn-ingress (the WireGuard remote-access VPN, clusters/base/vpn-ingress.yaml) carries
|
||||
# platform/vpn-ingress/wg-server-key.sops.yaml — the WireGuard SERVER private key, the one
|
||||
# secret in that component. Without this stanza kustomize hands Flux the raw ciphertext and the
|
||||
# Secret applies full of ENC[...]: the APPLY SUCCEEDS and every Flux condition stays green, and
|
||||
# the only symptom is a VPN that no client can handshake with. The DaemonSet's init container
|
||||
# validates the value with `wg pubkey` specifically so that state is a named CrashLoop instead.
|
||||
# The `sops-age` Secret is out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vpn-ingress
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# vuln-scanner (the CVE visibility scanner, clusters/base/vuln-scanner.yaml) carries
|
||||
# platform/vuln-scanner/vuln-scanner-oauth2-proxy-credentials.sops.yaml — the per-pod oauth2-proxy
|
||||
# client and cookie secrets. Without this stanza the Secret applies full of ENC[...] and the
|
||||
# sidecar cannot authenticate; the rest of the scanner runs regardless. `sops-age` is out-of-band.
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: vuln-scanner
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# cloudflared (the Cloudflare Tunnel connector, clusters/base/cloudflared.yaml) carries
|
||||
# platform/cloudflared/cloudflared-credentials.sops.yaml — the tunnel credentials JSON, the one
|
||||
# secret in that component. Without this stanza the Secret applies full of ENC[...]: the APPLY
|
||||
# SUCCEEDS, the Kustomization stays Ready, and the only symptom is a connector that cannot
|
||||
# authenticate to the edge (CloudflaredTunnelDown fires). The `sops-age` Secret is out-of-band
|
||||
# (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# pm-agent (the Program Manager agent service, clusters/base/pm-agent.yaml) carries
|
||||
# pm-agent-credentials.sops.yaml — the Gitea write token for the dedicated non-admin bot
|
||||
# account and the Anthropic subscription token from `claude setup-token`. Without this
|
||||
# stanza kustomize hands Flux the raw SOPS document and the Secret applies full of
|
||||
# ciphertext: the APPLY SUCCEEDS, and the only symptom is a dispatcher that cannot
|
||||
# authenticate to Gitea — so this stanza is as load-bearing as the file it decrypts (same
|
||||
# silent-failure shape as the apps-vms note below/above). The `sops-age` Secret is created
|
||||
# out-of-band (name only).
|
||||
- patch: |-
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: pm-agent
|
||||
namespace: flux-system
|
||||
spec:
|
||||
decryption:
|
||||
provider: sops
|
||||
secretRef:
|
||||
name: sops-age
|
||||
# apps-vms (the KubeVirt VM tier, clusters/base/apps-vms.yaml) gained its FIRST encrypted Secret
|
||||
# on 2026-09-14: apps/vms/unifi/unifi-uos-installer.sops.yaml, the operator's UOS installer URL,
|
||||
# which 30-virtualmachine.yaml mounts into the guest as a disk. Without this stanza kustomize
|
||||
@@ -503,60 +572,48 @@ patches:
|
||||
path: /data/LAN_CIDR
|
||||
value: "192.168.2.0/24"
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# BULK (EXTERNAL NFS) STORAGE HOST — *** NOT YET DECIDED. THIS PLACEHOLDER IS LOAD-BEARING. ***
|
||||
# BULK (EXTERNAL NFS) STORAGE HOST — the TrueNAS at 192.168.2.30.
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Backs every `class: bulk` volume: the 8.33 T media library (Storage/plex — plex, sonarr,
|
||||
# radarr) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
|
||||
# radarr, qbittorrent) and the 270 G legacy Nextcloud archive (Storage/nextcloud).
|
||||
#
|
||||
# THE STORAGE HOST IS NOT REACHABLE FROM PROD AT ALL RIGHT NOW. It has one interface,
|
||||
# vmbr0 192.168.1.9/24, on pfSense's LAN; prod's nodes are on 192.168.2.0/24, pfSense's WAN.
|
||||
# 22 and 8006 answer from prod only because pfSense port-forwards them via 192.168.2.102;
|
||||
# 2049 and 111 do not. Opening that path is an OPERATOR decision, and the two candidate
|
||||
# outcomes give DIFFERENT values here:
|
||||
# port-forward 192.168.2.102:2049 -> 192.168.1.9:2049 => "192.168.2.102"
|
||||
# a WAN->LAN route to 192.168.1.0/24 => "192.168.1.9"
|
||||
# Guessing either one authors a prod cluster that mounts nothing, or — worse, if the guess
|
||||
# ever became reachable — mounts something unintended. So it stays a placeholder.
|
||||
# THIS WAS THE REPO'S ONE LOAD-BEARING PLACEHOLDER, AND THE REASON IT COULD BE IS NOW GONE.
|
||||
# The datasets lived on a Proxmox host with a single interface on pfSense's LAN
|
||||
# (192.168.1.9), which prod could reach only through a DNAT at 192.168.2.102 — so the correct
|
||||
# value here depended on an undecided operator choice, and guessing it would have authored a
|
||||
# cluster that mounts nothing. Both pools were physically moved onto a standalone TrueNAS at
|
||||
# 192.168.2.30, prod's OWN /24. No crossing to arrange, no candidates to choose between.
|
||||
# Measured 2026-10-07: 192.168.2.102:2049 no longer answers; 192.168.2.30:2049 and :111 do.
|
||||
#
|
||||
# WHICH PORTS THE DECISION MUST OPEN: exactly ONE, tcp/2049. The PVs mount nfsvers=4.1, and
|
||||
# NFSv4 carries mount and lock in-protocol — no rpcbind/111, no mountd, no statd. Choosing v4
|
||||
# over v3 was in part this decision's cost: v3 would have needed four ports and three of them
|
||||
# on ephemeral numbers.
|
||||
#
|
||||
# WHAT THE EXPORT MUST ALLOW IS ALREADY SETTLED, and it is the same under BOTH outcomes.
|
||||
# Measured 2026-09-07 from this workstation (192.168.2.132) through the existing port-forward:
|
||||
# the storage host logged the connection as coming FROM 192.168.2.132, not from a pfSense LAN
|
||||
# address (`ss -tn` peer 192.168.2.132, and the host's only route off-LAN is its default via
|
||||
# 192.168.1.1). pfSense DNATs WITHOUT source NAT, so the NFS server will see prod's REAL node
|
||||
# addresses either way. The additive `zfs set sharenfs=` values are in docs/BULK-NFS-STORAGE.md.
|
||||
#
|
||||
# *** BLAST RADIUS — READ BEFORE MERGING THIS TO A BRANCH PROD RECONCILES. ***
|
||||
# A PersistentVolume's spec.nfs.server is an unvalidated string: the API server ACCEPTS this
|
||||
# placeholder, the PV is created and the PVC BINDS to it (binding is pure control-plane). The
|
||||
# failure surfaces only when a pod mounts — MountVolume.SetUp "Failed to resolve server
|
||||
# TBD-PROD-BULK-NFS-SERVER" — so plex, sonarr, radarr and nextcloud would sit in
|
||||
# ContainerCreating on prod. Unlike the LAN knobs above, this placeholder does NOT fail at
|
||||
# apply time and there is no CRD validation to lean on. Set this value BEFORE prod reconciles
|
||||
# a revision containing the bulk volumes; the dev overlay renders a working value, which is
|
||||
# where these manifests are meant to be proven first.
|
||||
# ONE PORT IS STILL ALL IT NEEDS: tcp/2049. The PVs mount nfsvers=4.1 and NFSv4 carries mount
|
||||
# and lock in-protocol — no rpcbind/111, no mountd, no statd. The NAS advertises
|
||||
# `+3 +4 +4.1 +4.2` (/proc/fs/nfsd/versions, measured), so 4.1 negotiates.
|
||||
- op: replace
|
||||
path: /data/BULK_NFS_SERVER
|
||||
# 192.168.2.102 is pfSense's WAN address, which DNATs tcp/2049 through to the Proxmox
|
||||
# host's REAL address 192.168.1.9 on pfSense's LAN. Prod sits on the WAN side and cannot
|
||||
# reach 192.168.1.9 directly, so the cluster mounts the NAT address. Dev is on the LAN and
|
||||
# keeps the base value 192.168.1.9.
|
||||
# VERIFIED ON THE NAS 2026-10-07, before any cluster touched it (`exportfs -v`):
|
||||
# /mnt/Storage/plex 192.168.2.10/.11/.12 rw,sec=sys,secure,root_squash,no_all_squash
|
||||
# /mnt/Storage/nextcloud 192.168.2.10/.11/.12 ro,sec=sys,secure,root_squash,no_all_squash
|
||||
#
|
||||
# PROVEN from k3s-prod-1 on 2026-09-12, not inferred:
|
||||
# - mount of 192.168.2.102:/mnt/Storage/plex negotiated nfs4 vers=4.1 proto=tcp
|
||||
# - a write as uid 1000 succeeded. Tested as 1000 and NOT root deliberately: the export
|
||||
# carries no_root_squash, so a root write would have passed and proved nothing, while
|
||||
# LSIO images default to PUID 911 and would have gone Ready then failed every import
|
||||
# with EACCES. The tree is 0755 1000:3000, so uid 1000 is the only writer.
|
||||
# - ln() across downloads/ -> tv/ returned link count 2 (same inode), so the arrs get
|
||||
# real atomic moves instead of silently degrading to copy-then-delete on EXDEV.
|
||||
# pfSense DNATs WITHOUT source NAT (the host logs prod's own node addresses as the peer),
|
||||
# so the ZFS export ACL names 192.168.2.10/.11/.12 directly rather than one translated IP.
|
||||
value: "192.168.2.102"
|
||||
# no_all_squash is the flag the entire permission model rests on: client uids pass through
|
||||
# untouched, so the arrs' PUID=1000 arrives as the owner of a 0755 1000:3000 tree. sec=sys
|
||||
# keeps that numeric rather than principal-mapped. The export deliberately does NOT carry
|
||||
# no_root_squash, which the Proxmox one did: nothing here needs it, because every write to
|
||||
# the tree happens after the LSIO drop to uid 1000 and the images only chown their Longhorn
|
||||
# /config. Both subPath directories (plex/downloads, nextcloud/nextcloud) already exist at
|
||||
# 0755 1000:3000, so a squashed root can still traverse them for subPath resolution.
|
||||
#
|
||||
# The ro= posture on the archive is now SERVER-enforced, which the old export also managed.
|
||||
# The media tree's read-only half (plex) stays a CLIENT-side guarantee — all four media pods
|
||||
# can land on any node, so the server sees the same three addresses for reader and writers
|
||||
# alike and cannot tell them apart. Unchanged by the move; see docs/BULK-NFS-STORAGE.md §5.
|
||||
#
|
||||
# *** CHANGING THIS VALUE REQUIRES DELETING AND RECREATING THE PVs. *** A bound PV's
|
||||
# spec.nfs is immutable — measured by server dry-run 2026-10-07:
|
||||
# spec.persistentvolumesource: Forbidden: spec.persistentvolumesource is immutable after creation
|
||||
# so Flux's apply FAILS rather than updating and the pods keep their old, dead mount. The
|
||||
# operator sequence is docs/BULK-NFS-STORAGE.md §10. Retain + the renderer's pre-set
|
||||
# claimRef make it safe: no data is touched and each recreated PV rebinds its own PVC.
|
||||
value: "192.168.2.30"
|
||||
# ==========================================================================================
|
||||
# OFF-CLUSTER DR TARGET — Cloudflare R2 (operator, 2026-09-05; bucket + token 2026-09-07).
|
||||
# ==========================================================================================
|
||||
|
||||
@@ -98,7 +98,10 @@ RUN set -eux; \
|
||||
|
||||
# --- runtime -----------------------------------------------------------------
|
||||
ARG FEDORA_VERSION=44
|
||||
FROM registry.fedoraproject.org/fedora-minimal:${FEDORA_VERSION}
|
||||
# Stage `conformance-runner` is the image that ships; stage `verify` (end of file) is a TEST stage
|
||||
# built FROM it that only runs assertions and is never pushed. build-image.sh and the CI
|
||||
# pipeline build `--target verify` to prove it and `--target conformance-runner` for the image.
|
||||
FROM registry.fedoraproject.org/fedora-minimal:${FEDORA_VERSION} AS conformance-runner
|
||||
|
||||
ARG CHAINSAW_VERSION=
|
||||
ARG KUBECTL_VERSION=
|
||||
@@ -198,3 +201,86 @@ WORKDIR /workspace
|
||||
# prepend something the policy cannot see. `bash` is the interactive fallback for a human
|
||||
# debugging the image locally.
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
# =============================================================================
|
||||
# verify — a TEST stage, never shipped. Every assertion conformance/incluster/build-image.sh used
|
||||
# to make with `run --user 65532 --read-only --tmpfs /workspace` now runs here, at build time,
|
||||
# because CI builds on rootless BuildKit where no container runtime exists.
|
||||
#
|
||||
# WHAT A BUILD STEP CAN AND CANNOT REPRODUCE:
|
||||
# * uid 65532 ............ exact (USER inherited from the conformance-runner stage).
|
||||
# * no network ........... exact: RUN --network=none, and the first check proves it.
|
||||
# * --read-only rootfs ... NOT reproducible (a RUN step always has a writable layer).
|
||||
# Approximated by permission: everything uid 65532 can write must lie
|
||||
# under /workspace, /tmp or /var/tmp. In the Job only /workspace (and
|
||||
# /results) are writable — the image points HOME and TMPDIR there — and
|
||||
# readOnlyRootFilesystem is enforced by
|
||||
# platform/kyverno/admission/conformance-trigger-job-shape.yaml.
|
||||
# =============================================================================
|
||||
FROM conformance-runner AS verify
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for the Job's isolation) ---"
|
||||
if getent hosts gitea.home.keithjackson.dev >/dev/null 2>&1 || timeout 3 bash -c '</dev/tcp/1.1.1.1/443' 2>/dev/null; then
|
||||
echo "UNEXPECTED: network reachable in the verify stage"; exit 1
|
||||
fi
|
||||
echo "confirmed: no network"
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME}"
|
||||
echo "--- bash ---"; bash --version | head -1
|
||||
echo "--- python3 ---"; python3 --version
|
||||
echo "--- git ---"; git --version
|
||||
echo "--- kubectl ---"; kubectl version --client
|
||||
echo "--- chainsaw ---"; chainsaw version
|
||||
echo "--- baked versions ---"; cat /etc/conformance-runner-versions
|
||||
echo "--- conformance-clone: fixed path, root-owned, NOT writable by the runner ---"
|
||||
stat -c "%n %U:%G %a" /usr/local/bin/conformance-clone
|
||||
if [ -w /usr/local/bin/conformance-clone ]; then
|
||||
echo "UNEXPECTED: uid $(id -u) can write the one executable the policy will trust"; exit 1
|
||||
fi
|
||||
echo "confirmed: not writable by uid $(id -u)"
|
||||
# Each of these MUST fail. Assert the non-zero exit as well as the message: a check that only
|
||||
# greps stderr would pass a build where the guard had stopped rejecting.
|
||||
expect_fail() {
|
||||
local label="$1"; shift
|
||||
local out rc=0
|
||||
out="$(env "$@" 2>&1)" || rc=$?
|
||||
echo "--- ${label} ---"
|
||||
printf "%s\n" "${out}" | head -2
|
||||
if [ "${rc}" -eq 0 ]; then
|
||||
echo "UNEXPECTED: that should have failed, but it exited 0"; exit 1
|
||||
fi
|
||||
echo "(rejected, exit ${rc})"
|
||||
}
|
||||
expect_fail "conformance-clone rejects arguments (that is where an injection would live)" \
|
||||
conformance-clone --layer l2
|
||||
expect_fail "conformance-clone rejects a missing SHA" \
|
||||
conformance-clone
|
||||
expect_fail "conformance-clone rejects a branch name instead of a SHA (no silent run of the tip)" \
|
||||
CONFORMANCE_GIT_SHA=main conformance-clone
|
||||
expect_fail "conformance-clone rejects an UNRENDERED placeholder" \
|
||||
CONFORMANCE_GIT_SHA=@@CONFORMANCE_GIT_SHA@@ conformance-clone
|
||||
expect_fail "conformance-clone rejects a HOME off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
HOME=/etc conformance-clone
|
||||
expect_fail "conformance-clone refuses a REDIRECTED repository URL (the env-injection hole)" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_REPO_URL=https://evil.invalid/x.git conformance-clone
|
||||
expect_fail "conformance-clone refuses a clone target off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_SRC=/etc/src conformance-clone
|
||||
echo "--- the root filesystem is not writable by the runner (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: / is writable by uid $(id -u)"; exit 1; fi
|
||||
echo "confirmed: / is not writable"
|
||||
echo "--- everything uid $(id -u) CAN write lies under /workspace, /tmp or /var/tmp ---"
|
||||
stray="$(find / -xdev \( -path /proc -o -path /sys -o -path /dev \) -prune -o -writable ! -type l -print 2>/dev/null \
|
||||
| grep -vE '^/(workspace|tmp|var/tmp)(/|$)' || true)"
|
||||
if [ -n "${stray}" ]; then echo "UNEXPECTED: writable outside the allowed paths:"; printf ' %s\n' ${stray} | head -20; exit 1; fi
|
||||
echo "confirmed: no writable path outside /workspace, /tmp, /var/tmp"
|
||||
echo "--- HOME is writable (the workspace emptyDir) ---"
|
||||
touch "${HOME}/.probe" || { echo "UNEXPECTED: ${HOME} not writable"; exit 1; }
|
||||
echo "confirmed: ${HOME} is writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] || { echo "UNEXPECTED: running as root"; exit 1; }
|
||||
echo "confirmed: running as uid $(id -u), not root"
|
||||
echo "CONFORMANCE-RUNNER VERIFIED"
|
||||
VERIFY
|
||||
@@ -61,10 +61,15 @@ credentials — the Job's own scoped ServiceAccount does (R-SEC-02).
|
||||
> digest-pinned to the conformance-runner image, `command` exactly
|
||||
> `/usr/local/bin/conformance-clone`, and **no `args` at all**. `ephemeralContainers` and
|
||||
> `env`/`envFrom` were the same shape of hole and are closed in the same policy.
|
||||
- `build-image.sh` — builds the above, **verifies the toolchain by running it** inside the
|
||||
image as uid 65532 with `--read-only`, and prints the digest. It **never pushes**: it
|
||||
prints the exact `podman push --digestfile …` command for the operator instead. See
|
||||
`conformance/versions.env` `CONFORMANCE_RUNNER_IMAGE` for the post-push pin.
|
||||
- `build-image.sh` — builds the above and **verifies the toolchain by running it**: the
|
||||
Containerfile's `verify` stage executes every tool and every `conformance-clone` refusal as
|
||||
uid 65532 with no network, at build time. It **never pushes**, and neither does anyone from a
|
||||
workstation: CI (`.gitea/workflows/images.yaml`) builds the same Containerfile and `verify`
|
||||
stage on rootless BuildKit (`platform/buildkit/`), and on a push to `main` publishes the
|
||||
verified image under `sha-<commit>` and `main`. Renovate then moves
|
||||
`CONFORMANCE_RUNNER_IMAGE_DIGEST` in `conformance/versions.env` onto the registry-returned
|
||||
digest in a pull request (that file is a sensitive path, so the bump needs an
|
||||
administrator's `/approve-override`).
|
||||
`--verify-clone` additionally has `conformance-clone` really clone this repo at a real SHA
|
||||
and asserts the checkout matches (opt-in, because it needs the Gitea host reachable — the
|
||||
default verify stays offline so an air-gapped build is fully checked, not half-checked).
|
||||
|
||||
@@ -1,43 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# conformance/incluster/build-image.sh — build the pinned `conformance-runner`
|
||||
# image and print its digest. Closes docs/plans/l2-l3-conformance-ci-wiring.md §3 item 1.
|
||||
# image and prove it works. Closes docs/plans/l2-l3-conformance-ci-wiring.md §3 item 1.
|
||||
#
|
||||
# ./build-image.sh # build + verify + print the digest and push command
|
||||
# ./build-image.sh # build the verify target, then the image
|
||||
# ./build-image.sh --print-build-args # print KEY=VALUE build args (what CI consumes)
|
||||
# ./build-image.sh --no-verify # build only (skip the in-image tool proof)
|
||||
# ./build-image.sh --verify-clone # ALSO really clone this repo with conformance-clone.
|
||||
# # Opt-in because it needs the Gitea host reachable; the
|
||||
# # default verify is offline so an air-gapped build is
|
||||
# # still fully checked rather than half-checked.
|
||||
# ENGINE=docker ./build-image.sh # build with docker instead of podman
|
||||
# IMAGE_TAG=foo ./build-image.sh # override the local tag (CI uses this)
|
||||
#
|
||||
# THIS SCRIPT NEVER PUSHES. Publishing is an operator action: the registry credential is
|
||||
# not an agent's to hold, and the pushed digest is what a fail-closed admission policy
|
||||
# then trusts. So this builds locally, proves the toolchain works INSIDE the image as the
|
||||
# non-root uid the policy demands, and prints the exact `podman push` line for a human.
|
||||
# THIS SCRIPT NEVER PUSHES — and nobody pushes this image from a workstation either.
|
||||
# Publishing is done by CI: .gitea/workflows/images.yaml builds the SAME Containerfile on
|
||||
# rootless BuildKit (platform/buildkit/), taking its build args from `--print-build-args`, and
|
||||
# on a push to `main` pushes it to the Gitea registry and records the registry-returned digest
|
||||
# in the job summary; Renovate then opens the pull request that moves conformance/versions.env
|
||||
# onto that digest. The proof that the image works is the Containerfile's `verify` stage, which
|
||||
# this script (podman locally) and CI both build.
|
||||
#
|
||||
# VERSIONS ARE NOT DEFINED HERE. They are read from infra/devcontainer/versions.env, the
|
||||
# single source of truth (CHAINSAW_VERSION, KUBECTL_VERSION, FEDORA_VERSION) — the same
|
||||
# pins the toolbox gate runs with. Nothing in this file invents a version.
|
||||
#
|
||||
# THE DIGEST CAVEAT — read it before pasting a digest into anything.
|
||||
# The digest printed here is the LOCAL manifest digest. It is normally identical to the
|
||||
# registry digest (podman pushes the manifest it already has rather than recompressing),
|
||||
# but that is a property of the push, not a guarantee. So the push command below uses
|
||||
# `--digestfile`: the digest the registry actually returns is the authoritative one, and
|
||||
# that is the one that goes into conformance/versions.env.
|
||||
# NO DIGEST IS PRINTED. A manifest digest does not exist until a registry holds the manifest,
|
||||
# and the only digest anything may pin is the one the REGISTRY returns. CI captures that at
|
||||
# push time (.gitea/workflows/scripts/images.sh) and Renovate carries it into
|
||||
# conformance/versions.env.
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
||||
ENGINE="${ENGINE:-podman}"
|
||||
TAG_OVERRIDE="${IMAGE_TAG:-}"
|
||||
|
||||
VERIFY=1
|
||||
VERIFY_CLONE=0
|
||||
PRINT_ARGS=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-verify) VERIFY=0 ;;
|
||||
--print-build-args) PRINT_ARGS=1 ;;
|
||||
--verify-clone) VERIFY_CLONE=1 ;;
|
||||
-h|--help) sed -n '2,/^####/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
||||
@@ -48,8 +54,6 @@ log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- pins: sourced, never invented -------------------------------------------
|
||||
TOOLCHAIN_ENV="${REPO_ROOT}/infra/devcontainer/versions.env"
|
||||
[ -f "${TOOLCHAIN_ENV}" ] || die "missing ${TOOLCHAIN_ENV} — the pins live there, not here"
|
||||
@@ -59,14 +63,21 @@ source "${TOOLCHAIN_ENV}"
|
||||
: "${KUBECTL_VERSION:?KUBECTL_VERSION not set in ${TOOLCHAIN_ENV}}"
|
||||
: "${FEDORA_VERSION:?FEDORA_VERSION not set in ${TOOLCHAIN_ENV}}"
|
||||
|
||||
# THE build args, in ONE place; `--print-build-args` hands them to the CI pipeline, which builds
|
||||
# this Containerfile on rootless BuildKit.
|
||||
BUILD_ARGS=("CHAINSAW_VERSION=${CHAINSAW_VERSION}" "KUBECTL_VERSION=${KUBECTL_VERSION}" "FEDORA_VERSION=${FEDORA_VERSION}")
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
command -v "${ENGINE}" >/dev/null 2>&1 || die "'${ENGINE}' not on PATH (set ENGINE=docker to use docker)"
|
||||
|
||||
# --- image name: fixed by the admission policy -------------------------------
|
||||
# platform/kyverno/admission/conformance-trigger-job-shape.yaml admits ONLY
|
||||
# gitea.home.keithjackson.dev/home-network/conformance-runner@sha256:*. The repository path
|
||||
# is therefore not a preference; a different path is a DENIED Job. Do not "improve" it here.
|
||||
IMAGE_REPO="gitea.home.keithjackson.dev/home-network/conformance-runner"
|
||||
# The tag is build convenience only (a human-readable handle for the push). Admission
|
||||
# ignores it: only the digest is ever referenced from a Job.
|
||||
IMAGE_TAG="chainsaw${CHAINSAW_VERSION}-kubectl${KUBECTL_VERSION}-fc${FEDORA_VERSION}"
|
||||
# The tag is a local handle only. CI re-tags what it publishes (git SHA + `main`), and
|
||||
# admission ignores tags entirely: only the digest is ever referenced from a Job.
|
||||
IMAGE_TAG="${TAG_OVERRIDE:-chainsaw${CHAINSAW_VERSION}-kubectl${KUBECTL_VERSION}-fc${FEDORA_VERSION}}"
|
||||
IMAGE_REF="${IMAGE_REPO}:${IMAGE_TAG}"
|
||||
|
||||
log "Building ${IMAGE_REF}"
|
||||
@@ -76,86 +87,31 @@ info " CHAINSAW ${CHAINSAW_VERSION}"
|
||||
info " KUBECTL ${KUBECTL_VERSION}"
|
||||
info " FEDORA ${FEDORA_VERSION}"
|
||||
|
||||
"${ENGINE}" build \
|
||||
--file "${HERE}/Containerfile" \
|
||||
--tag "${IMAGE_REF}" \
|
||||
--build-arg "CHAINSAW_VERSION=${CHAINSAW_VERSION}" \
|
||||
--build-arg "KUBECTL_VERSION=${KUBECTL_VERSION}" \
|
||||
--build-arg "FEDORA_VERSION=${FEDORA_VERSION}" \
|
||||
"${HERE}"
|
||||
# Provenance: CI passes the commit it is building, so a pulled image names its source tree.
|
||||
LABEL_ARGS=()
|
||||
[ -z "${BUILD_REVISION:-}" ] || LABEL_ARGS+=(--label "org.opencontainers.image.revision=${BUILD_REVISION}")
|
||||
|
||||
ARG_FLAGS=()
|
||||
for a in "${BUILD_ARGS[@]}"; do ARG_FLAGS+=(--build-arg "$a"); done
|
||||
|
||||
# --- verify: the point of the whole exercise ---------------------------------
|
||||
# The stock chainsaw image "had" chainsaw and still could not run ./run.sh. So prove the
|
||||
# tools EXECUTE, and prove they execute as the NON-ROOT uid with a READ-ONLY root
|
||||
# filesystem — the shape the admission policy forces. A pass here is the only evidence
|
||||
# this image is usable; a build that merely succeeded is not.
|
||||
# The stock chainsaw image "had" chainsaw and still could not run ./run.sh. So the proof is that
|
||||
# the tools EXECUTE, as the NON-ROOT uid, with no network, that conformance-clone is not writable
|
||||
# by that uid and refuses every hostile input, and that nothing outside /workspace is writable.
|
||||
# That proof is the Containerfile's `verify` stage, built here and in CI alike (CI has no
|
||||
# container runtime — rootless BuildKit only). See the stage's header for what a build step can
|
||||
# and cannot reproduce of the Job's shape. A build that merely succeeded is not evidence.
|
||||
if [ "${VERIFY}" -eq 1 ]; then
|
||||
log "Verifying the toolchain inside the image (non-root, read-only rootfs)"
|
||||
"${ENGINE}" run --rm \
|
||||
--user 65532:65532 \
|
||||
--read-only \
|
||||
--tmpfs /workspace:rw,mode=1777 \
|
||||
--entrypoint "" \
|
||||
"${IMAGE_REF}" \
|
||||
/bin/bash -euo pipefail -c '
|
||||
echo "--- id ---"; id
|
||||
echo "--- HOME ---"; echo "HOME=${HOME}"
|
||||
echo "--- bash ---"; bash --version | head -1
|
||||
echo "--- python3 ---"; python3 --version
|
||||
echo "--- git ---"; git --version
|
||||
echo "--- kubectl ---"; kubectl version --client
|
||||
echo "--- chainsaw ---"; chainsaw version
|
||||
echo "--- baked versions ---"; cat /etc/conformance-runner-versions
|
||||
echo "--- conformance-clone: fixed path, root-owned, NOT writable by the runner ---"
|
||||
stat -c "%n %U:%G %a" /usr/local/bin/conformance-clone
|
||||
if [ -w /usr/local/bin/conformance-clone ]; then
|
||||
echo "UNEXPECTED: uid $(id -u) can write the one executable the policy will trust"; exit 1
|
||||
fi
|
||||
echo "confirmed: not writable by uid $(id -u)"
|
||||
# Each of these MUST fail. Assert the non-zero exit as well as the message: a check
|
||||
# that only greps stderr would pass a build where the guard had stopped rejecting.
|
||||
# (`|| true` inside the capture keeps `set -e`/`pipefail` from aborting on the
|
||||
# expected failure — without it the verify dies at the first negative case.)
|
||||
expect_fail() {
|
||||
local label="$1"; shift
|
||||
local out rc=0
|
||||
out="$(env "$@" 2>&1)" || rc=$?
|
||||
echo "--- ${label} ---"
|
||||
printf "%s\n" "${out}" | head -2
|
||||
if [ "${rc}" -eq 0 ]; then
|
||||
echo "UNEXPECTED: that should have failed, but it exited 0"; exit 1
|
||||
fi
|
||||
echo "(rejected, exit ${rc})"
|
||||
}
|
||||
expect_fail "conformance-clone rejects arguments (that is where an injection would live)" \
|
||||
conformance-clone --layer l2
|
||||
expect_fail "conformance-clone rejects a missing SHA" \
|
||||
conformance-clone
|
||||
expect_fail "conformance-clone rejects a branch name instead of a SHA (no silent run of the tip)" \
|
||||
CONFORMANCE_GIT_SHA=main conformance-clone
|
||||
expect_fail "conformance-clone rejects an UNRENDERED placeholder" \
|
||||
CONFORMANCE_GIT_SHA=@@CONFORMANCE_GIT_SHA@@ conformance-clone
|
||||
expect_fail "conformance-clone rejects a HOME off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
HOME=/etc conformance-clone
|
||||
expect_fail "conformance-clone refuses a REDIRECTED repository URL (the env-injection hole)" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_REPO_URL=https://evil.invalid/x.git conformance-clone
|
||||
expect_fail "conformance-clone refuses a clone target off the workspace volume" \
|
||||
CONFORMANCE_GIT_SHA=0000000000000000000000000000000000000000 \
|
||||
CONFORMANCE_SRC=/etc/src conformance-clone
|
||||
echo "--- rootfs is read-only (expected: a failure) ---"
|
||||
if touch /should-fail 2>/dev/null; then echo "UNEXPECTED: rootfs is writable"; exit 1; fi
|
||||
echo "confirmed: / is read-only"
|
||||
echo "--- HOME is writable (the workspace emptyDir) ---"
|
||||
touch "${HOME}/.probe" && echo "confirmed: ${HOME} is writable"
|
||||
echo "--- uid is non-zero ---"
|
||||
[ "$(id -u)" -ne 0 ] && echo "confirmed: running as uid $(id -u), not root"
|
||||
'
|
||||
log "Verifying the toolchain (Containerfile target: verify)"
|
||||
"${ENGINE}" build "${ARG_FLAGS[@]}" --target verify --file "${HERE}/Containerfile" "${HERE}"
|
||||
log "Toolchain verified"
|
||||
fi
|
||||
|
||||
"${ENGINE}" build "${LABEL_ARGS[@]}" "${ARG_FLAGS[@]}" --target conformance-runner \
|
||||
--file "${HERE}/Containerfile" --tag "${IMAGE_REF}" "${HERE}"
|
||||
|
||||
# --- verify the clone helper against the REAL repository (opt-in: needs network) ----
|
||||
# LOCAL ONLY: this one `run`s the image, which needs a container engine, so CI does not do it.
|
||||
# The negative cases above prove conformance-clone refuses bad input. This proves the
|
||||
# positive case: it lands the EXACT commit asked for and can prove it did. Without this, the
|
||||
# rev-parse assertion is only an unexercised line of shell.
|
||||
@@ -184,46 +140,12 @@ if [ "${VERIFY_CLONE}" -eq 1 ]; then
|
||||
log "conformance-clone verified against the real repository"
|
||||
fi
|
||||
|
||||
# --- digest ------------------------------------------------------------------
|
||||
DIGEST="$("${ENGINE}" image inspect --format '{{.Digest}}' "${IMAGE_REF}")"
|
||||
# --- result ------------------------------------------------------------------
|
||||
IMAGE_ID="$("${ENGINE}" image inspect --format '{{.Id}}' "${IMAGE_REF}")"
|
||||
SIZE="$("${ENGINE}" image inspect --format '{{.Size}}' "${IMAGE_REF}")"
|
||||
|
||||
log "Built ${IMAGE_REF}"
|
||||
info "local manifest digest ${DIGEST}"
|
||||
info "image id ${IMAGE_ID}"
|
||||
info "size $(( SIZE / 1024 / 1024 )) MiB"
|
||||
|
||||
cat <<EOF
|
||||
|
||||
================================================================================
|
||||
NOT PUSHED. Publishing is an operator action — this script holds no registry
|
||||
credential and will never acquire one.
|
||||
|
||||
OPERATOR: run these two commands, then paste the digest from the digestfile into
|
||||
conformance/versions.env (CONFORMANCE_RUNNER_IMAGE) via a PR:
|
||||
|
||||
${ENGINE} login gitea.home.keithjackson.dev
|
||||
|
||||
${ENGINE} push --digestfile /tmp/conformance-runner.digest \\
|
||||
${IMAGE_REF}
|
||||
|
||||
Then:
|
||||
|
||||
cat /tmp/conformance-runner.digest
|
||||
# -> sha256:.... THIS is the authoritative digest, not the local one above.
|
||||
|
||||
# conformance/versions.env
|
||||
CONFORMANCE_RUNNER_IMAGE="${IMAGE_REPO}@\$(cat /tmp/conformance-runner.digest)"
|
||||
|
||||
WHY THE DIGEST AND NOT THE TAG: platform/kyverno/admission/conformance-trigger-job-shape.yaml
|
||||
is a fail-closed Enforce policy. It DENIES any conformance-trigger Job whose image is not
|
||||
${IMAGE_REPO}@sha256:<digest>. A tag reference is denied,
|
||||
and so is any other repository path.
|
||||
|
||||
TIGHTEN THE BOUNDARY AFTERWARDS: that policy's header asks for the wildcard
|
||||
\`${IMAGE_REPO}@sha256:*\` to be replaced with the
|
||||
LITERAL digest once the image exists, closing the "anyone who can push to this path" gap.
|
||||
That is a separate, human-reviewed PR against the security boundary.
|
||||
================================================================================
|
||||
EOF
|
||||
info "image id ${IMAGE_ID}"
|
||||
info "size $(( SIZE / 1024 / 1024 )) MiB"
|
||||
info "Not pushed. CI publishes this image from main (.gitea/workflows/images.yaml) and"
|
||||
info "Renovate moves conformance/versions.env onto the registry-returned digest."
|
||||
@@ -14,7 +14,7 @@ Pure standard library (matches the harness's no-pip-dependency posture); the che
|
||||
themselves shell out (render checks to catalog/render.py, whose PyYAML dependency lives
|
||||
there; L0 checks to git + PyYAML).
|
||||
|
||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] <test-dir> [<test-dir> ...]
|
||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] [--jobs N] <test-dir> [<test-dir> ...]
|
||||
Exit 0 iff every selected offline check passes.
|
||||
|
||||
The per-dir check script is whichever of `render-check.sh` / `check.sh` the directory
|
||||
@@ -43,13 +43,35 @@ Exit codes: 0 all selected checks passed / 1 at least one FAILED / 3 nothing fai
|
||||
scope-declared check was UNWIRED. conformance/run.sh propagates 3 verbatim; ci/l0.sh turns it
|
||||
into the gate's own rc 3. This is the ONE place the "unwired, not green" rule lives now,
|
||||
instead of once per hand-written stage.
|
||||
|
||||
PARALLELISM (--jobs, env CONF_OFFLINE_JOBS)
|
||||
-------------------------------------------
|
||||
The checks are independent processes, so they run in a bounded thread pool (one `bash` child
|
||||
per worker). Default: min(8, usable CPUs); `--jobs 1` is the old serial runner exactly. Each
|
||||
check's stdout+stderr is CAPTURED and only reported after every check has finished, in the
|
||||
ORDER THE DIRS WERE GIVEN, so the log, the JUnit and the step summary are byte-for-byte
|
||||
independent of which check finished first (only the trailing "[N.Ns]" wall-time on each
|
||||
status line varies). The 0/1/3 aggregation runs over the same ordered case list as before.
|
||||
|
||||
Parallel-safety is a property of the CHECKS, not of this runner: every check must keep its
|
||||
scratch state in a private mktemp/mkdtemp dir and must not write into the tree or a fixed
|
||||
path. The audit that established that for the current set is recorded in
|
||||
decisions/2026-11-08-parallel-l0-checks.md; a new check that needs shared state must say so.
|
||||
|
||||
Nested pools: the keycloak reconciler checks run their own mutation suites in a pool
|
||||
(mutations.py, env MUTATION_JOBS). When this runner is itself parallel it exports a SMALLER
|
||||
MUTATION_JOBS (unless the caller already set one), so the two suites share the CPUs the
|
||||
other jobs-2 checks are not using instead of each taking min(8, CPUs). See inner_jobs().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from xml.dom import minidom
|
||||
@@ -97,6 +119,46 @@ def collect_meta(test_yaml: Path, prefix: str):
|
||||
return name, reqs
|
||||
|
||||
|
||||
def usable_cpus() -> int:
|
||||
"""CPUs this process may run on (the cgroup/affinity view, not the host's count)."""
|
||||
try:
|
||||
return max(1, len(os.sched_getaffinity(0)))
|
||||
except (AttributeError, OSError):
|
||||
return max(1, os.cpu_count() or 1)
|
||||
|
||||
|
||||
def positive_int(text: str) -> int:
|
||||
n = int(text)
|
||||
if n < 1:
|
||||
raise argparse.ArgumentTypeError(f"must be >= 1, got {n}")
|
||||
return n
|
||||
|
||||
|
||||
def inner_jobs(cpus: int, outer: int) -> int:
|
||||
"""MUTATION_JOBS to hand the checks when this runner runs `outer` of them at once.
|
||||
|
||||
Two checks (the keycloak client-secret and passkey reconcilers) fan out internally; the
|
||||
other outer-2 workers are single processes. Give the two suites an equal share of the CPUs
|
||||
those others leave free, never fewer than 2 each (a 1-wide pool would make the 40-mutation
|
||||
passkey suite the critical path of the whole stage), never more than 8. Peak process count
|
||||
is then about (outer-2) + 2*inner, i.e. <= cpus on any machine with cpus >= outer + 2, and
|
||||
at worst ~1.25-1.5x cpus on a small runner — CPU-bound oversubscription of that size costs
|
||||
latency per process, not throughput.
|
||||
"""
|
||||
if outer <= 1:
|
||||
return min(8, cpus)
|
||||
return max(2, min(8, (cpus - outer + 2) // 2))
|
||||
|
||||
|
||||
def run_check(check: Path, env: dict) -> tuple[int, str, float]:
|
||||
t0 = time.monotonic()
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True,
|
||||
stdin=subprocess.DEVNULL, env=env,
|
||||
)
|
||||
return proc.returncode, (proc.stdout + proc.stderr).strip(), time.monotonic() - t0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--junit", type=Path, required=True)
|
||||
@@ -108,26 +170,54 @@ def main() -> int:
|
||||
"(makes this runner exit 3); repeatable")
|
||||
ap.add_argument("--summary-out", type=Path, default=None,
|
||||
help="write one Markdown line per case here, for the CI step summary")
|
||||
ap.add_argument("--jobs", type=positive_int, default=None,
|
||||
help="checks to run at once (default: env CONF_OFFLINE_JOBS, else "
|
||||
"min(8, usable CPUs)); 1 = serial")
|
||||
ap.add_argument("dirs", nargs="+", type=Path)
|
||||
args = ap.parse_args()
|
||||
|
||||
strict_scopes = set(args.unwired_scope)
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||
cpus = usable_cpus()
|
||||
if args.jobs is not None:
|
||||
jobs = args.jobs
|
||||
elif os.environ.get("CONF_OFFLINE_JOBS", "").strip():
|
||||
try:
|
||||
jobs = positive_int(os.environ["CONF_OFFLINE_JOBS"].strip())
|
||||
except (ValueError, argparse.ArgumentTypeError) as e:
|
||||
ap.error(f"CONF_OFFLINE_JOBS={os.environ['CONF_OFFLINE_JOBS']!r}: {e}")
|
||||
else:
|
||||
jobs = min(8, cpus)
|
||||
child_env = dict(os.environ)
|
||||
if not child_env.get("MUTATION_JOBS", "").strip():
|
||||
child_env["MUTATION_JOBS"] = str(inner_jobs(cpus, jobs))
|
||||
print(f"[offline] running {len(args.dirs)} check(s), jobs={jobs} "
|
||||
f"(cpus={cpus}, MUTATION_JOBS={child_env['MUTATION_JOBS']})", file=sys.stderr)
|
||||
|
||||
# Metadata first, in the given order; then the checks themselves through the pool. The
|
||||
# results are consumed in SUBMISSION order, so `cases` is identical to the serial runner's.
|
||||
planned = [] # (name, reqs, dir, scope, check-or-None)
|
||||
for d in args.dirs:
|
||||
check = next((d / s for s in CHECK_SCRIPTS if (d / s).exists()), None)
|
||||
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
||||
scope = scope_of(d / "chainsaw-test.yaml", args.scope_annotation)
|
||||
if check is None:
|
||||
cases.append((name, reqs, d, 3,
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||
scope))
|
||||
continue
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True
|
||||
)
|
||||
out = (proc.stdout + proc.stderr).strip()
|
||||
cases.append((name, reqs, d, proc.returncode, out, scope))
|
||||
planned.append((name, reqs, d, scope, check))
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||
elapsed = [] # wall seconds per case, same order (None: nothing ran)
|
||||
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
||||
futures = [pool.submit(run_check, check, child_env) if check is not None else None
|
||||
for (_n, _r, _d, _s, check) in planned]
|
||||
for (name, reqs, d, scope, check), fut in zip(planned, futures):
|
||||
if fut is None:
|
||||
cases.append((name, reqs, d, 3,
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||
scope))
|
||||
elapsed.append(None)
|
||||
continue
|
||||
rc, out, secs = fut.result()
|
||||
cases.append((name, reqs, d, rc, out, scope))
|
||||
elapsed.append(secs)
|
||||
|
||||
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
||||
skipped = sum(1 for c in cases if c[3] == 3)
|
||||
@@ -145,7 +235,7 @@ def main() -> int:
|
||||
},
|
||||
)
|
||||
summary_lines = []
|
||||
for name, reqs, d, rc, out, scope in cases:
|
||||
for (name, reqs, d, rc, out, scope), secs in zip(cases, elapsed):
|
||||
tc = ET.SubElement(
|
||||
suite,
|
||||
"testcase",
|
||||
@@ -164,8 +254,9 @@ def main() -> int:
|
||||
fail.text = out
|
||||
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
||||
mark = f" scope={scope}" if scope else ""
|
||||
took = f" [{secs:.1f}s]" if secs is not None else ""
|
||||
print(f"[offline] {layer_of(d):3s} {status:4s} {name} "
|
||||
f"({','.join(reqs) or 'untagged'}){mark}",
|
||||
f"({','.join(reqs) or 'untagged'}){mark}{took}",
|
||||
file=sys.stderr)
|
||||
if rc not in (0, 3):
|
||||
print(out, file=sys.stderr)
|
||||
|
||||
@@ -135,6 +135,44 @@ requirements:
|
||||
safety_critical: false
|
||||
summary: "Update policy per-service configurable (cadence, channel, pinning)."
|
||||
|
||||
# --- Vulnerability visibility ---
|
||||
- id: R-VULN-01
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Inventory every running container image by digest with its workloads, from the cluster."
|
||||
- id: R-VULN-02
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Every running image evaluated against a current database at least every 24h; new images within 2h."
|
||||
- id: R-VULN-03
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Any target not evaluated is visibly reported with its reason; absence is never shown as clean."
|
||||
- id: R-VULN-04
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Authenticated page lists every CVE with affected images, containers and workloads; searchable and filterable."
|
||||
- id: R-VULN-05
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Inventory, scanning, reporting and triage run on a schedule with no human or AI action; loud failure."
|
||||
- id: R-VULN-06
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Scan failure, staleness and unaccepted CRITICAL fixes reported by email as grouped alerts."
|
||||
- id: R-VULN-07
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Scheduled bot keeps one dashboard issue and requests remediation via Renovate; no CVE detail exposed."
|
||||
- id: R-VULN-08
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Accepted risk declared in git with reason and mandatory expiry; reverts to open on expiry."
|
||||
- id: R-VULN-09
|
||||
doc: docs/17-vulnerability-visibility.md
|
||||
safety_critical: false
|
||||
summary: "Scanner digest-pinned, no auto-merged updates, no Secret read or write access, no git credential."
|
||||
|
||||
# --- Observability & alerting ---
|
||||
- id: R-OBS-01
|
||||
doc: docs/10-observability-and-alerting.md
|
||||
|
||||
@@ -363,6 +363,102 @@ SCRAPE_OWNERS: list[dict] = [
|
||||
"why": "velero_* powers VeleroBackupFailure / PartialFailure / NoRecentBackup — the rules "
|
||||
"that failed to notice prod running with zero volume backups",
|
||||
},
|
||||
# ── ADDED 2026-10-17 (feat/automated-upgrades). A NEW OWNER, NOT A RELAXED ASSERTION. ──────
|
||||
# This entry is purely additive: it registers a new metric family and its enablement, and it
|
||||
# changes nothing about how any existing owner is evaluated. It makes the check STRICTER for the
|
||||
# component it covers, not looser — it adds two `require` patterns plus a `companion` obligation
|
||||
# that platform/kured must keep satisfying, so a future edit that disables kured's Service,
|
||||
# ServiceMonitor or NetworkPolicy FAILS the gate instead of silently re-inerting three alerts.
|
||||
#
|
||||
# WHY THE CHECK HAD TO BE TOUCHED AT ALL, rather than the manifest being wrong: assertion (3)
|
||||
# fails any alert over a metric no owner claims, and the escape hatch (KNOWN_UNWIRED_METRICS) is
|
||||
# SHRINK-ONLY, so declaring `kured_*` as accepted debt is not permitted and would be the wrong
|
||||
# answer anyway — the scrape genuinely IS wired here. The only two honest options were to
|
||||
# register the owner (this) or to ship the reboot automation with no alerting at all, which for a
|
||||
# component whose every failure mode is silent would have been the worse engineering choice.
|
||||
# `conformance/**` is a sensitive approval path precisely so that this reasoning gets read.
|
||||
{
|
||||
"owner": "kured: service.create + metrics.create in the HelmRelease",
|
||||
"prefixes": ("kured_",),
|
||||
"file": "platform/kured/helmrelease.yaml",
|
||||
# BOTH keys are required and the chart defaults BOTH to false. `metrics.create` makes the
|
||||
# ServiceMonitor; `service.create` makes the Service it selects. Enable only the monitor and
|
||||
# it selects nothing, so the single kured metric is never scraped — and a PrometheusRule over
|
||||
# a series no target produces is INERT, which for this component means its own liveness
|
||||
# alarm is the thing that breaks. Matched as two separate anchored blocks rather than one
|
||||
# pattern because they are separate top-level values keys, not nested.
|
||||
"require": [
|
||||
r"service:\s*\n\s*create:\s*true",
|
||||
r"metrics:\s*\n\s*create:\s*true",
|
||||
],
|
||||
"forbid": [],
|
||||
"why": "kured_reboot_required is the ONLY metric kured exposes (one GaugeVec, "
|
||||
"cmd/kured/main.go:92-96) and it powers all three platform-health.node-reboots "
|
||||
"rules. Those rules are the only way to learn that OS reboot automation has stalled: "
|
||||
"kured stays 1/1 Running while rebooting nothing, so a missed scrape here is "
|
||||
"indistinguishable from a cluster that needed no reboots",
|
||||
# The ServiceMonitor ALONE is not sufficient — the same NetworkPolicy trap Longhorn hit.
|
||||
"companion": {
|
||||
"file": "platform/kured/kustomization.yaml",
|
||||
"require": [r"^\s*-\s*networkpolicy\.yaml\s*$"],
|
||||
"why": "platform/kured/networkpolicy.yaml ships a default-deny baseline (the kured "
|
||||
"namespace is a platform namespace, so Kyverno's app-namespace-scoped generator "
|
||||
"does not cover it) AND the ingress allow that lets Prometheus reach :8080 "
|
||||
"through it. Drop the networkpolicy from this kustomization and the deny-all "
|
||||
"goes too, so this currently fails SAFE — but reinstating the baseline without "
|
||||
"the allow reproduces Longhorn's measured failure exactly: targets DISCOVERED, "
|
||||
"every scrape 'context deadline exceeded', TSDB empty, all three rules quiet. "
|
||||
"Keeping the two in one file makes that pair inseparable",
|
||||
},
|
||||
},
|
||||
# ── ADDED 2026-11-09 (CVE visibility, ADR-0027). A NEW OWNER, NOT A RELAXED ASSERTION. ──────
|
||||
# Registers the vuln_scanner_* family and the Service+ServiceMonitor that scrapes it.
|
||||
{
|
||||
"owner": "vuln-scanner: Service + ServiceMonitor on port `metrics` (:9090)",
|
||||
"prefixes": ("vuln_scanner_",),
|
||||
"file": "platform/vuln-scanner/servicemonitor.yaml",
|
||||
"require": [
|
||||
r"^kind:\s*ServiceMonitor\s*$",
|
||||
r"^\s*-\s*port:\s*metrics\s*$",
|
||||
],
|
||||
"forbid": [],
|
||||
"why": "vuln_scanner_* powers the five VulnScan alerts, the only email path for scan "
|
||||
"failure, staleness and overdue critical fixes (R-VULN-06). Two of those rules fire "
|
||||
"on an absent metric, so a lost scrape is loud rather than inert",
|
||||
"companion": {
|
||||
"file": "platform/vuln-scanner/kustomization.yaml",
|
||||
"require": [r"^\s*-\s*networkpolicy\.yaml\s*$"],
|
||||
"why": "platform/vuln-scanner/networkpolicy.yaml ships the namespace's default-deny "
|
||||
"and the ingress allow for Prometheus on :9090 in one file, so a baseline "
|
||||
"without its allow cannot empty the TSDB",
|
||||
},
|
||||
},
|
||||
# ── ADDED 2026-11-08 (feat/cloudflare-tunnel). A NEW OWNER, NOT A RELAXED ASSERTION. ───────
|
||||
# Purely additive, the same shape as the kured entry: it registers the cloudflared_* family
|
||||
# and the Service+ServiceMonitor that scrapes it, and makes the check STRICTER for that component.
|
||||
{
|
||||
"owner": "cloudflared: headless metrics Service + ServiceMonitor on port `metrics` (:2000)",
|
||||
"prefixes": ("cloudflared_",),
|
||||
"file": "platform/cloudflared/servicemonitor.yaml",
|
||||
"require": [
|
||||
r"^kind:\s*ServiceMonitor\s*$",
|
||||
r"^kind:\s*Service\s*$",
|
||||
r"^\s*-\s*port:\s*metrics\s*$",
|
||||
],
|
||||
"forbid": [],
|
||||
"why": "cloudflared_tunnel_ha_connections powers CloudflaredTunnelDown, the only signal "
|
||||
"that the tunnel has lost the Cloudflare edge. That rule is written to FIRE on an "
|
||||
"absent series, so a lost scrape is loud rather than inert — but it would then be "
|
||||
"permanently firing for a wiring reason, which teaches the operator to ignore it",
|
||||
"companion": {
|
||||
"file": "platform/cloudflared/kustomization.yaml",
|
||||
"require": [r"^\s*-\s*networkpolicy\.yaml\s*$"],
|
||||
"why": "platform/cloudflared/networkpolicy.yaml ships the namespace's default-deny "
|
||||
"(a platform namespace, outside Kyverno's generator) together with the ingress "
|
||||
"allow for Prometheus on :2000. Keeping them in one file, applied together, is "
|
||||
"what stops a baseline-without-allow from emptying the TSDB",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# `up` is produced by EVERY scrape target, so no owner can meaningfully claim it and assertion (1)
|
||||
|
||||
@@ -29,7 +29,8 @@
|
||||
# THEN (1) promtool is a PINNED member of the toolchain: PROMTOOL_VERSION is declared in BOTH
|
||||
# infra/devcontainer/versions.env and ci/versions.env and the two AGREE (the repo's
|
||||
# lockstep rule), the Containerfile takes it as a build ARG, installs it from the
|
||||
# pinned prometheus release and records it for provenance, dev.sh passes it, and the
|
||||
# pinned prometheus release and records it for provenance, build-image.sh (the one
|
||||
# recipe `dev.sh build` and CI both use) passes it, and the
|
||||
# CI runner installs it for the l0 stage;
|
||||
# (2) the promtool ACTUALLY IN USE reports exactly that pinned version — a unit test is
|
||||
# evidence only if the engine matches the one prod evaluates rules with;
|
||||
@@ -246,10 +247,12 @@ check('echo "promtool=${PROMTOOL_VERSION}"' in cf_text,
|
||||
"infra/devcontainer/Containerfile does not record promtool in /etc/dev-toolchain-versions — "
|
||||
"that provenance line is how an operator tells a stale image from a current one")
|
||||
|
||||
DEVSH = ROOT / "infra/devcontainer/dev.sh"
|
||||
check(DEVSH.is_file() and "--build-arg PROMTOOL_VERSION=" in DEVSH.read_text(),
|
||||
"infra/devcontainer/dev.sh does not pass --build-arg PROMTOOL_VERSION — the ARG would be "
|
||||
"empty and the build would fetch a bogus URL")
|
||||
# The build recipe moved from dev.sh into build-image.sh on 2026-10-29, so that `dev.sh build`
|
||||
# and the CI image pipeline (.gitea/workflows/images.yaml) share ONE recipe. Check the recipe.
|
||||
BUILDSH = ROOT / "infra/devcontainer/build-image.sh"
|
||||
check(BUILDSH.is_file() and '"PROMTOOL_VERSION=$PROMTOOL_VERSION"' in BUILDSH.read_text(),
|
||||
"infra/devcontainer/build-image.sh does not pass PROMTOOL_VERSION in its BUILD_ARGS — the ARG "
|
||||
"would be empty and the build would fetch a bogus URL")
|
||||
|
||||
INSTALL = ROOT / ".gitea/workflows/scripts/install-tools.sh"
|
||||
if INSTALL.is_file():
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the Cloudflare Tunnel exposes only what the operator decided, and cannot stall Flux
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# platform/cloudflared/config.yaml is the entire internet-facing surface of the tunnel. This test
|
||||
# asserts the operator decisions of 2026-11-08 against it on every gate run: no rule may target
|
||||
# anything but a 404 or the shared Gateway (so never the Kubernetes API); the LAN/VPN-only
|
||||
# hostnames (plex, jackett, pihole, semaphore, qbittorrent, radarr, sonarr, headlamp,
|
||||
# unifi, crafty-mc) 404 before the wildcard; the
|
||||
# egress policy admits no API/node entity; and the Flux tier is isolated (wait:false, nothing
|
||||
# dependsOn it, both overlays decrypt it). It proves itself non-vacuous by mutation. Rationale and
|
||||
# the honest NOT-proven list: check.sh.
|
||||
#
|
||||
# WHY SCOPE, NOT REQUIREMENTS: the property spans platform/cloudflared, platform/gateway (the
|
||||
# derived Service name), clusters/base and both overlays. A requirement-scoped selection would skip
|
||||
# it on, e.g., a Gateway rename — precisely a change that breaks it.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: cloudflared-ingress-rules
|
||||
labels:
|
||||
conformance.home.k3s/R-NET-01: "true"
|
||||
conformance.home.k3s/R-NET-02: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-NET-01, R-NET-02"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
The Cloudflare Tunnel routes only to the shared Gateway or a 404, never to the Kubernetes
|
||||
API; the LAN/VPN-only hostnames 404 ahead of the wildcard; the connector's egress admits no
|
||||
API or node entity; and its Flux tier cannot stall any other tier.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-tunnel-ingress-rules-and-isolation
|
||||
description: >-
|
||||
Parse platform/cloudflared/config.yaml with BASE_DOMAIN substituted, assert every rule's
|
||||
target against the Gateway Service derived from platform/gateway/gateway.yaml, assert the
|
||||
excluded hostnames 404 first, check the egress CiliumNetworkPolicy and the Flux wiring, and
|
||||
run the evaluator on mutated configs to prove it is not vacuous; and assert the
|
||||
credentials Secret volume is readable by the container's run-as uid/groups (fsGroup +
|
||||
mode), self-tested by mutation.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — cloudflared-ingress-rules
|
||||
# Serves: R-NET-01 / R-NET-02 (default-deny; allow rules declarative and scoped), and the four
|
||||
# operator decisions of 2026-11-08 recorded in decisions/2026-11-08-cloudflare-tunnel.md.
|
||||
#
|
||||
# WHY THIS EXISTS. platform/cloudflared/config.yaml is the whole of what the internet can reach
|
||||
# through the Cloudflare Tunnel. A one-line edit there — a rule pointing at the Kubernetes API, a
|
||||
# LAN-only hostname moved below the wildcard, a dropped catch-all — publishes something the operator
|
||||
# decided must never be public, and nothing in the cluster turns red: the tunnel stays connected and
|
||||
# the Kustomization stays Ready. So the decisions are asserted here, on every gate run.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) every ingress rule's `service` is `http_status:404` or EXACTLY the shared Gateway's
|
||||
# in-cluster Service, derived from platform/gateway/gateway.yaml (so a Gateway rename
|
||||
# fails here rather than in production) — which by construction excludes the
|
||||
# Kubernetes API, and a named deny-list says so explicitly when it is violated;
|
||||
# (2) each LAN/VPN-only hostname (plex, plex-vip, jackett, pihole, pihole-dns, semaphore,
|
||||
# qbittorrent, radarr, sonarr, headlamp, unifi, crafty-mc) has a rule mapping it to http_status:404 that comes BEFORE any
|
||||
# Gateway-bound rule (cloudflared matches first-rule-wins);
|
||||
# (3) the apex and the wildcard are the Gateway rules, with matchSNItoHost, and nothing
|
||||
# disables origin TLS verification; the last rule is a hostname-less 404 catch-all;
|
||||
# (4) the egress CiliumNetworkPolicy admits none of kube-apiserver/host/remote-node/
|
||||
# cluster/all and no CIDR rule, and its backend selector excludes the LAN-only
|
||||
# namespaces (the second, independent layer behind (2));
|
||||
# (5) the Flux tier is ISOLATED: its own Kustomization, wait:false, not listed in
|
||||
# platform/kustomization.yaml, nothing anywhere dependsOn it, and both overlays carry its
|
||||
# SOPS decryption patch — so a missing or invalid credential can stall nothing else;
|
||||
# (6) the rule evaluator is not vacuous: it is run against mutated copies of the real config
|
||||
# (an API rule added, plex moved below the wildcard, the catch-all removed) and must
|
||||
# flag each one;
|
||||
# AND (7) every Secret volume the container mounts — the tunnel credentials above all — is
|
||||
# READABLE by the container's run-as identity, modelled the way the kubelet lays it out:
|
||||
# files owned by uid 0, group-owned by fsGroup if set (else gid 0), with the item mode or
|
||||
# defaultMode (API default 0644). The identity is the container securityContext over
|
||||
# the pod's; the process's groups are runAsGroup + fsGroup + supplementalGroups. An
|
||||
# unset runAsUser is a FAIL, not a pass: the image default cannot be read offline. The
|
||||
# model is deliberately conservative — it does not credit the kubelet's habit of
|
||||
# OR-ing group-read into fsGroup-owned volumes, so 0400+fsGroup fails here though it
|
||||
# would work. Self-tested by mutation (fsGroup removed; mode 0400; runAsUser unset).
|
||||
# This is the 2026-11-08 CrashLoop: runAsUser 65532, no fsGroup, 0440 root:root.
|
||||
#
|
||||
# NOT proven (static): that Cloudflare routes only the hostnames whose DNS points at the tunnel,
|
||||
# that the Gateway's certificate validates from cloudflared, or that Envoy enforces (4) — those are
|
||||
# runtime properties, verified by the operator per runbooks/cloudflare-tunnel.md.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("CHECK FAIL (cloudflared-ingress-rules): PyYAML is required", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES, CHECKS = [], 0
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def docs(path):
|
||||
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
||||
|
||||
|
||||
COMP = ROOT / "platform" / "cloudflared"
|
||||
CONFIG = COMP / "config.yaml"
|
||||
if not CONFIG.is_file():
|
||||
print(f"CHECK FAIL (cloudflared-ingress-rules): {CONFIG.relative_to(ROOT)} is absent — the "
|
||||
"component this check guards was moved or deleted; re-point or delete this check in the "
|
||||
"SAME change", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
# The hostnames the operator decided never go through the tunnel (decisions 2026-11-08).
|
||||
# headlamp: operator decision 2026-11-08 (LAN/VPN-only). qbittorrent/radarr/sonarr: catalog
|
||||
# `exposure: internal`, like jackett/pihole/semaphore.
|
||||
EXCLUDED_HOSTS = ("plex", "plex-vip", "jackett", "pihole", "pihole-dns", "semaphore",
|
||||
"qbittorrent", "radarr", "sonarr", "headlamp", "cve", "unifi", "crafty-mc")
|
||||
# Their Gateway-backend namespaces (unifi / crafty-mc are L4 VIPs, not Gateway backends).
|
||||
EXCLUDED_NAMESPACES = ("plex", "jackett", "pihole", "semaphore", "qbittorrent", "radarr",
|
||||
"sonarr", "headlamp", "vuln-scanner")
|
||||
FORBIDDEN_ENTITIES = {"kube-apiserver", "host", "remote-node", "cluster", "all"}
|
||||
# Any of these in a `service` means a route toward the Kubernetes API (or a raw-TCP/SSH bastion).
|
||||
API_MARKERS = ("kubernetes", "kube-apiserver", "apiserver", ":6443", "10.43.0.1", "tcp://",
|
||||
"ssh://", "unix:", "bastion")
|
||||
|
||||
# ── substitution: ${BASE_DOMAIN} from cluster-vars, the ONLY token allowed ──────────────────────
|
||||
cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml")
|
||||
if d.get("kind") == "ConfigMap")
|
||||
base_domain = cv["data"]["BASE_DOMAIN"]
|
||||
raw = CONFIG.read_text()
|
||||
tokens = set(re.findall(r"\$\{([A-Za-z0-9_]+)\}", raw))
|
||||
check(tokens == {"BASE_DOMAIN"},
|
||||
f"(0) config.yaml uses exactly the ${{BASE_DOMAIN}} token (found {sorted(tokens)}); any other "
|
||||
f"token is substituted from cluster-vars unseen by this check")
|
||||
check(re.search(r"\$(?!\{BASE_DOMAIN\})", raw) is None,
|
||||
"(0) config.yaml contains a bare `$` — Flux envsubst would rewrite or empty it")
|
||||
cfg = yaml.safe_load(raw.replace("${BASE_DOMAIN}", base_domain))
|
||||
|
||||
# ── the Gateway Service, derived rather than hard-coded ─────────────────────────────────────────
|
||||
gw = next(d for d in docs(ROOT / "platform" / "gateway" / "gateway.yaml") if d.get("kind") == "Gateway")
|
||||
GATEWAY_SVC = (f"https://cilium-gateway-{gw['metadata']['name']}."
|
||||
f"{gw['metadata']['namespace']}.svc.cluster.local:443")
|
||||
|
||||
|
||||
def evaluate(cfg):
|
||||
"""Return the list of rule violations for a parsed cloudflared config."""
|
||||
bad = []
|
||||
rules = cfg.get("ingress") or []
|
||||
if len(rules) < len(EXCLUDED_HOSTS) + 3:
|
||||
bad.append(f"only {len(rules)} ingress rules — the scan is vacuous")
|
||||
return bad
|
||||
for i, r in enumerate(rules):
|
||||
svc = str(r.get("service", ""))
|
||||
if any(m in svc.lower() for m in API_MARKERS):
|
||||
bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}, which "
|
||||
f"is a route toward the Kubernetes API / a raw bastion — the API stays "
|
||||
f"local-only (operator decision 2026-11-08)")
|
||||
if svc not in ("http_status:404", GATEWAY_SVC):
|
||||
bad.append(f"rule {i} ({r.get('hostname', '<catch-all>')}) targets {svc!r}; only "
|
||||
f"http_status:404 or {GATEWAY_SVC} are permitted")
|
||||
if (r.get("originRequest") or {}).get("noTLSVerify"):
|
||||
bad.append(f"rule {i} disables origin TLS verification (noTLSVerify)")
|
||||
if (cfg.get("originRequest") or {}).get("noTLSVerify"):
|
||||
bad.append("top-level originRequest disables origin TLS verification (noTLSVerify)")
|
||||
|
||||
gw_idx = [i for i, r in enumerate(rules) if r.get("service") == GATEWAY_SVC]
|
||||
first_gw = min(gw_idx) if gw_idx else len(rules)
|
||||
for h in EXCLUDED_HOSTS:
|
||||
fqdn = f"{h}.{base_domain}"
|
||||
hits = [i for i, r in enumerate(rules) if r.get("hostname") == fqdn]
|
||||
if not hits:
|
||||
bad.append(f"no rule for LAN/VPN-only host {fqdn}; it would fall through to the "
|
||||
f"wildcard and reach the Gateway")
|
||||
continue
|
||||
i = hits[0]
|
||||
if rules[i].get("service") != "http_status:404":
|
||||
bad.append(f"{fqdn} maps to {rules[i].get('service')!r}, not http_status:404")
|
||||
if i > first_gw:
|
||||
bad.append(f"{fqdn}'s 404 rule (#{i}) comes AFTER the first Gateway rule (#{first_gw}); "
|
||||
f"cloudflared is first-match, so the Gateway rule wins")
|
||||
|
||||
gw_hosts = {rules[i].get("hostname") for i in gw_idx}
|
||||
if gw_hosts != {base_domain, f"*.{base_domain}"}:
|
||||
bad.append(f"the Gateway-bound hostnames are {sorted(map(str, gw_hosts))}; expected "
|
||||
f"exactly the apex and the wildcard")
|
||||
for i in gw_idx:
|
||||
if (rules[i].get("originRequest") or {}).get("matchSNItoHost") is not True:
|
||||
bad.append(f"Gateway rule #{i} lacks originRequest.matchSNItoHost: true — the Gateway "
|
||||
f"would not see the SNI that selects its wildcard certificate")
|
||||
last = rules[-1]
|
||||
if "hostname" in last or last.get("service") != "http_status:404":
|
||||
bad.append("the last rule is not a hostname-less http_status:404 catch-all")
|
||||
return bad
|
||||
|
||||
|
||||
# (1)-(3) the real config
|
||||
for v in evaluate(cfg):
|
||||
check(False, f"(1-3) {v}")
|
||||
check(str(cfg.get("credentials-file", "")).startswith("/etc/cloudflared/creds/"),
|
||||
"(3) credentials-file points into the mounted Secret volume")
|
||||
|
||||
# (6) mutation self-test — the evaluator must flag each broken variant
|
||||
mutants = {}
|
||||
m = copy.deepcopy(cfg)
|
||||
m["ingress"].insert(0, {"hostname": f"k8s.{base_domain}",
|
||||
"service": "https://kubernetes.default.svc:443"})
|
||||
mutants["API rule added"] = m
|
||||
m = copy.deepcopy(cfg)
|
||||
plex = next(r for r in m["ingress"] if r.get("hostname") == f"plex.{base_domain}")
|
||||
m["ingress"].remove(plex)
|
||||
m["ingress"].insert(len(m["ingress"]) - 1, plex)
|
||||
mutants["plex moved below the wildcard"] = m
|
||||
m = copy.deepcopy(cfg)
|
||||
m["ingress"] = m["ingress"][:-1]
|
||||
mutants["catch-all removed"] = m
|
||||
for name, mut in mutants.items():
|
||||
check(evaluate(mut), f"(6) self-test: the evaluator did NOT flag the mutant '{name}' — the "
|
||||
f"check is vacuous")
|
||||
|
||||
# (4) the egress policy
|
||||
np_docs = docs(COMP / "networkpolicy.yaml")
|
||||
cnps = [d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"]
|
||||
check(len(cnps) == 1, f"(4) exactly one CiliumNetworkPolicy in networkpolicy.yaml (found {len(cnps)})")
|
||||
for cnp in cnps:
|
||||
for i, rule in enumerate(cnp["spec"].get("egress") or []):
|
||||
ents = set(rule.get("toEntities") or [])
|
||||
check(not (ents & FORBIDDEN_ENTITIES),
|
||||
f"(4) egress rule {i} admits {sorted(ents & FORBIDDEN_ENTITIES)} — that reaches the "
|
||||
f"Kubernetes API or the nodes; the API stays local-only")
|
||||
check(not (rule.get("toCIDR") or rule.get("toCIDRSet")),
|
||||
f"(4) egress rule {i} uses a CIDR — a CIDR can name the API VIP; use entities")
|
||||
for ep in rule.get("toEndpoints") or []:
|
||||
notin = [e for e in ep.get("matchExpressions") or []
|
||||
if e.get("key") == "io.kubernetes.pod.namespace" and e.get("operator") == "NotIn"]
|
||||
missing = set(EXCLUDED_NAMESPACES) - set(notin[0]["values"] if notin else [])
|
||||
check(not missing,
|
||||
f"(4) backend selector in egress rule {i} does not exclude {sorted(missing)}")
|
||||
for d in np_docs:
|
||||
if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny":
|
||||
check(set(d["spec"].get("policyTypes", [])) == {"Ingress", "Egress"},
|
||||
"(4) default-deny covers both directions")
|
||||
break
|
||||
else:
|
||||
check(False, "(4) networkpolicy.yaml ships no default-deny baseline for the namespace")
|
||||
|
||||
# (5) Flux isolation
|
||||
fk = next(d for d in docs(ROOT / "clusters" / "base" / "cloudflared.yaml")
|
||||
if d.get("kind") == "Kustomization")
|
||||
check(fk["spec"].get("wait") is False,
|
||||
"(5) clusters/base/cloudflared.yaml must be wait: false — its pods are legitimately not-Ready "
|
||||
"until the tunnel is enrolled")
|
||||
check(fk["spec"].get("path") == "./platform/cloudflared", "(5) the Kustomization reconciles ./platform/cloudflared")
|
||||
check("cloudflared.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(),
|
||||
"(5) clusters/base/kustomization.yaml lists cloudflared.yaml")
|
||||
check("cloudflared" not in (ROOT / "platform" / "kustomization.yaml").read_text(),
|
||||
"(5) platform/kustomization.yaml must not include cloudflared — that is the gating tier")
|
||||
for f in sorted((ROOT / "clusters").rglob("*.yaml")):
|
||||
for d in docs(f):
|
||||
for dep in (d.get("spec") or {}).get("dependsOn") or []:
|
||||
check(not (isinstance(dep, dict) and dep.get("name") == "cloudflared"),
|
||||
f"(5) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn "
|
||||
f"cloudflared — that hands an unenrolled tunnel the power to stall it")
|
||||
for env in ("dev", "prod"):
|
||||
text = (ROOT / "clusters" / env / "kustomization.yaml").read_text()
|
||||
check(re.search(r"name:\s*cloudflared\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n"
|
||||
r"\s*decryption:", text) is not None,
|
||||
f"(5) clusters/{env}/kustomization.yaml carries the cloudflared SOPS decryption patch")
|
||||
|
||||
dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment")
|
||||
check(dep["spec"].get("replicas", 1) >= 2, "(5) the Deployment runs at least 2 replicas")
|
||||
check(dep["spec"]["template"]["metadata"]["labels"].get("homelab.keithjackson.dev/egress-class")
|
||||
== "internet",
|
||||
"(4) the pods are egress-class=internet — vpn would put the edge connections behind PIA")
|
||||
check(dep["spec"]["template"]["spec"].get("automountServiceAccountToken") is False,
|
||||
"(4) the pods mount no ServiceAccount token")
|
||||
|
||||
# (7) the credentials Secret is readable by the run-as identity
|
||||
def unreadable_secret_volumes(pod_spec):
|
||||
"""Return a description per Secret volume the containers mount but cannot read."""
|
||||
bad = []
|
||||
psc = pod_spec.get("securityContext") or {}
|
||||
vols = {v["name"]: v for v in pod_spec.get("volumes") or [] if "secret" in v}
|
||||
for c in (pod_spec.get("initContainers") or []) + (pod_spec.get("containers") or []):
|
||||
csc = c.get("securityContext") or {}
|
||||
uid = csc.get("runAsUser", psc.get("runAsUser"))
|
||||
gid = csc.get("runAsGroup", psc.get("runAsGroup"))
|
||||
fs_group = psc.get("fsGroup")
|
||||
groups = {g for g in [gid, fs_group, *(psc.get("supplementalGroups") or [])]
|
||||
if g is not None}
|
||||
for vm in c.get("volumeMounts") or []:
|
||||
v = vols.get(vm["name"])
|
||||
if v is None:
|
||||
continue
|
||||
sec = v["secret"]
|
||||
if uid is None:
|
||||
bad.append(f"container {c['name']} mounts Secret volume {vm['name']} but sets no "
|
||||
f"runAsUser (pod or container) — readability cannot be proven offline")
|
||||
continue
|
||||
default = sec.get("defaultMode", 0o644)
|
||||
modes = [it.get("mode", default) for it in sec.get("items") or []] or [default]
|
||||
owner_gid = fs_group if fs_group is not None else 0
|
||||
for mode in modes:
|
||||
readable = ((uid == 0 and mode & 0o400) or (owner_gid in groups and mode & 0o040)
|
||||
or mode & 0o004)
|
||||
if not readable:
|
||||
bad.append(f"container {c['name']} (uid {uid}, groups {sorted(groups)}) cannot "
|
||||
f"read Secret volume {vm['name']} ({sec.get('secretName')}): files "
|
||||
f"are 0:{owner_gid} mode {mode:04o} — set pod fsGroup to a group "
|
||||
f"the container runs with, keeping 0440")
|
||||
return bad
|
||||
|
||||
|
||||
pod_spec = dep["spec"]["template"]["spec"]
|
||||
creds_vol = [v for v in pod_spec.get("volumes") or []
|
||||
if (v.get("secret") or {}).get("secretName") == "cloudflared-credentials"]
|
||||
check(len(creds_vol) == 1, "(7) the Deployment mounts Secret cloudflared-credentials as a volume")
|
||||
for v in unreadable_secret_volumes(pod_spec):
|
||||
check(False, f"(7) {v}")
|
||||
sec_mutants = {}
|
||||
m = copy.deepcopy(pod_spec)
|
||||
m["securityContext"].pop("fsGroup", None)
|
||||
m["securityContext"].pop("supplementalGroups", None)
|
||||
sec_mutants["fsGroup removed"] = m
|
||||
m = copy.deepcopy(pod_spec)
|
||||
for v in m["volumes"]:
|
||||
if "secret" in v:
|
||||
v["secret"]["defaultMode"] = 0o400
|
||||
for it in v["secret"].get("items") or []:
|
||||
it["mode"] = 0o400
|
||||
sec_mutants["mode 0400"] = m
|
||||
m = copy.deepcopy(pod_spec)
|
||||
m["securityContext"].pop("runAsUser", None)
|
||||
for c in m["containers"]:
|
||||
(c.get("securityContext") or {}).pop("runAsUser", None)
|
||||
sec_mutants["runAsUser unset"] = m
|
||||
for name, mut in sec_mutants.items():
|
||||
check(unreadable_secret_volumes(mut),
|
||||
f"(7) self-test: the readability model did NOT flag the mutant '{name}' — vacuous")
|
||||
|
||||
if CHECKS < 25:
|
||||
FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL (cloudflared-ingress-rules): {len(FAILURES)} of {CHECKS} assertions failed",
|
||||
file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS (cloudflared-ingress-rules): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -445,8 +445,21 @@ join = re.search(r"--server https://\{\{\s*k3s_api_host", K3S_TEXT)
|
||||
check(join is not None,
|
||||
"roles/k3s_server does not join secondaries with `--server https://{{ k3s_api_host }}` "
|
||||
"— without a join target the servers are separate single-node clusters")
|
||||
join_when = re.search(r"--server https://.*?\n(?:.*\n){0,10}?\s*when:\s*k3s_server_mode\s*==\s*'secondary'",
|
||||
K3S_TEXT, re.S)
|
||||
# LINE-BOUNDED ON PURPOSE -- DO NOT "SIMPLIFY" BACK TO `.*` + re.S.
|
||||
# This was `--server https://.*?\n(?:.*\n){0,10}?\s*when:...` with re.S, which made every `.`
|
||||
# match newlines too. That is nested unbounded repetition (a `{0,10}` of a quantifier that can each
|
||||
# swallow the whole document), so a FAILED or late match backtracks exponentially. It cost
|
||||
# **19m54s of CPU in CI** (measured 2026-10-18 via `docker top` on the live job container) and made
|
||||
# the whole merge gate take ~28 minutes instead of ~2 on any change touching `infra/**` -- because
|
||||
# `ci/touched-requirements.sh` maps `infra/*` to R-ENV-01, which pulls in this whole-repo check.
|
||||
# It was latent for as long as the regex existed and only crossed the threshold when
|
||||
# roles/k3s_server grew by ~100 lines (the kubelet graceful-shutdown config, PR #71): K3S_TEXT is
|
||||
# every *.yml in that role concatenated, so the role growing is the input growing.
|
||||
# `[^\n]*` cannot cross a line, so the `{0,10}` is a genuine ten-line window with nothing to
|
||||
# backtrack into. Measured on the same input: 0.2 ms, same verdict (match found).
|
||||
join_when = re.search(
|
||||
r"--server https://[^\n]*\n(?:[^\n]*\n){0,10}?[ \t]*when:[ \t]*k3s_server_mode[ \t]*==[ \t]*'secondary'",
|
||||
K3S_TEXT)
|
||||
check(join_when is not None,
|
||||
"the join install task is not gated on k3s_server_mode == 'secondary' — the founder "
|
||||
"would also try to join itself")
|
||||
|
||||
+4
-3
@@ -92,9 +92,10 @@
|
||||
# the NAME axis only. R-NET-04 / the edge port-forward posture owns the other.
|
||||
# - Anything about the seven non-catalog publishers' own correctness. The register records WHAT
|
||||
# each publishes so the claim is reviewable, which is all a static check can offer.
|
||||
# - Anything about the five `public` services' TARGET. They still resolve to the Gateway VIP
|
||||
# rather than a routable address; that is an open operator decision recorded in
|
||||
# platform/externaldns/helmrelease.yaml, and asserting a target here would hard-code a guess.
|
||||
# - That the public names RESOLVE to Cloudflare at runtime. check.sh (9) pins the TREE: the
|
||||
# Gateway target is the tunnel UUID in platform/cloudflared/config.yaml, proxying is on, the L4
|
||||
# `service` publishers opt out, and every name the tunnel 404s is unpublished. Whether Cloudflare
|
||||
# then answers anycast for them is a live read (`dig @1.1.1.1`), not a static one.
|
||||
# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
|
||||
@@ -26,7 +26,20 @@
|
||||
# KNOWN_NON_CATALOG_PUBLISHERS — shrink-only, stale entries FAIL;
|
||||
# (7) the check is LOAD-BEARING: whole-repo scope declared in chainsaw-test.yaml, honoured by
|
||||
# ci/l0.sh's --include-scope union, and ci/gate.sh blocks on an l0 failure. NO hand-written
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears).
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears);
|
||||
# (8) the external-dns HelmRelease declares `annotationPrefix` explicitly and it matches the
|
||||
# prefix of EVERY external-dns annotation in the tree and in catalog/render.py — added after
|
||||
# the 2026-11-07 v0.23.0 default-prefix change silently voided (1)-(3) at runtime;
|
||||
# (9) THE CLOUDFLARE TUNNEL WIRING (2026-11-08) — every public name is a PROXIED CNAME to the tunnel:
|
||||
# (a) the shared Gateway's `target` is EXACTLY `<tunnel:>.cfargotunnel.com`, the UUID read from
|
||||
# platform/cloudflared/config.yaml, and the HelmRelease sets `--cloudflare-proxied`;
|
||||
# (b) every `service`-source publisher carries `cloudflare-proxied: "false"` (L4 A records stay
|
||||
# DNS-only), and no HTTPRoute opts out of proxying (a DNS-only tunnel CNAME is dead);
|
||||
# (c) no hostname the tunnel maps to `http_status:404` has a public record — no undisowned route
|
||||
# names it, and the only `service`-source publishers of one are the declared L4 DNS-only
|
||||
# names (unifi, crafty-mc); conversely every disowned route's hostname IS 404'd by the
|
||||
# tunnel, so the "unpublished" and "refused at the tunnel" lists cannot drift apart;
|
||||
# (d) the operator's LAN/VPN-only names (plex, headlamp) are disowned everywhere they appear.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — it is STATIC, offline, cluster-free):
|
||||
# - That the record is gone from Cloudflare. Live read; belongs beside
|
||||
@@ -36,8 +49,11 @@
|
||||
# both the A and its `a-` TXT within ~60s) — but a manual proof is not a gate.
|
||||
# - That an internal name is UNREACHABLE. A port-forward plus a literal IP needs no DNS record.
|
||||
# This check owns the NAME axis only.
|
||||
# - Anything about the five public services' TARGET. They still answer with the Gateway VIP; the
|
||||
# routable-target decision is open and recorded in platform/externaldns/helmrelease.yaml.
|
||||
# - That the tunnel actually SERVES the public names, or that Cloudflare honoured `proxied`. (9)
|
||||
# proves the tree asks for a proxied CNAME to the right tunnel; the live answer (`dig @1.1.1.1`
|
||||
# returning Cloudflare anycast) is a runtime read. Nor does (8)/(9) prove the RUNNING controller
|
||||
# honours the prefix — a future chart bump that drops it would publish PROXIED A records at the
|
||||
# Gateway VIP; (8) only keeps the tree and the HelmRelease value consistent.
|
||||
# - It does not read ../home-network-k3s. The spec is read-only (R-TEST-06).
|
||||
#
|
||||
# Exit 0 = all assertions pass. Exit 1 = a failure. Exit 3 (honest SKIP, which the runner turns
|
||||
@@ -90,9 +106,18 @@ CHECKS = 0
|
||||
# ControllerValue = "dns-controller" -> the value that CLAIMS an object
|
||||
# source/annotations/processors.go IsControllerMismatch skips a source object when the key is
|
||||
# PRESENT and the value is NOT ControllerValue. So "disowned" == key present, value != claimed.
|
||||
CONTROLLER_KEY = "external-dns.alpha.kubernetes.io/controller"
|
||||
#
|
||||
# THE PREFIX IS NOT A CONSTANT OF NATURE. Every key external-dns reads is `<annotationPrefix><name>`,
|
||||
# and the prefix is a runtime flag whose DEFAULT moved in v0.22.0 (#6424: alpha -> GA, "with no
|
||||
# fallback"). On 2026-11-07 the chart bump to v0.23.0 made every key below invisible to the running
|
||||
# controller while this check — which only asserted the keys were PRESENT — stayed green. (8) now
|
||||
# ties ANNOTATION_PREFIX to the HelmRelease's explicit `annotationPrefix`.
|
||||
ANNOTATION_PREFIX = "external-dns.alpha.kubernetes.io/"
|
||||
CONTROLLER_KEY = ANNOTATION_PREFIX + "controller"
|
||||
CONTROLLER_CLAIMED = "dns-controller"
|
||||
HOSTNAME_KEY = "external-dns.alpha.kubernetes.io/hostname"
|
||||
HOSTNAME_KEY = ANNOTATION_PREFIX + "hostname"
|
||||
# Any key that looks like an external-dns annotation, under ANY prefix variant (alpha, GA, typos).
|
||||
EXTERNAL_DNS_KEY_RE = re.compile(r"^external-dns(\.[a-z0-9-]+)*\.kubernetes\.io/")
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
@@ -129,9 +154,12 @@ def docs_of(p: Path):
|
||||
# unadjudicated set cannot grow without a review event. Same ratchet reasoning as
|
||||
# KNOWN_EXTERNAL_MONITORS in servicemonitor-ports-resolve.
|
||||
#
|
||||
# EVERY ENTRY HERE IS A NAME THAT STILL ANSWERS WITH AN RFC1918 ADDRESS IN A PUBLIC ZONE. That is
|
||||
# the honest state after this change, and it is deliberate: converting them is an operator
|
||||
# decision per name (is grafana meant to be publicly resolvable at all?), not a renderer change,
|
||||
# WHAT THE ENTRIES PUBLISH NOW (2026-11-08). Every HTTPRoute entry that is not disowned publishes a
|
||||
# PROXIED CNAME to the Cloudflare Tunnel (the shared Gateway's `target`, assertion 9a), so no origin
|
||||
# address of ours is in the public zone for it. Headlamp is disowned (operator: LAN/VPN-only, 9d).
|
||||
# The two `service`-source entries still publish their RFC1918 LB VIP as a DNS-only A record —
|
||||
# deliberate (L4 ports cannot ride Cloudflare's HTTP proxy), and pinned DNS-only by 9b. Whether each
|
||||
# remaining name should be public at all stays an operator decision per name, not a renderer change,
|
||||
# and a cluster-wide ExternalDNS opt-in filter — the available hardening, see
|
||||
# platform/externaldns/helmrelease.yaml — would DELETE all of them at once the moment it is set.
|
||||
#
|
||||
@@ -139,27 +167,32 @@ def docs_of(p: Path):
|
||||
# =============================================================================================
|
||||
KNOWN_NON_CATALOG_PUBLISHERS: dict[str, str] = {
|
||||
"platform/oauth2-proxy/httproute.yaml:oauth2-proxy/oauth2-proxy":
|
||||
"publishes auth.${BASE_DOMAIN} -> the Gateway VIP. The oauth2-proxy login endpoint is a "
|
||||
"publishes auth.${BASE_DOMAIN} -> the tunnel (proxied CNAME). The oauth2-proxy login endpoint is a "
|
||||
"platform component, not a catalog service; it must be reachable by every browser that "
|
||||
"any forward-auth app redirects, so whether it is PUBLICLY resolvable is coupled to "
|
||||
"whether the five public services are — an operator decision, not a renderer one.",
|
||||
"platform/keycloak/httproute.yaml:keycloak/keycloak":
|
||||
"publishes keycloak.${BASE_DOMAIN} -> the Gateway VIP, and it is the route that actually "
|
||||
"publishes keycloak.${BASE_DOMAIN} -> the tunnel (proxied CNAME), and it is the route that actually "
|
||||
"OWNS that record (the TXT registry entry reads resource=httproute/keycloak/keycloak, not "
|
||||
"the catalog-rendered keycloak-8080 route, because external-dns merges duplicate "
|
||||
"hostnames and this one won). Keycloak is `exposure: public` in the catalog, so the name "
|
||||
"is meant to be public either way; it is listed because the PUBLISHING object is the "
|
||||
"hand-authored platform route, which carries no exposure field.",
|
||||
"platform/monitoring/grafana-httproute.yaml:monitoring/grafana":
|
||||
"publishes grafana.${BASE_DOMAIN} -> the Gateway VIP. An admin dashboard; almost "
|
||||
"certainly should not be publicly resolvable, but it is not a catalog service and "
|
||||
"withdrawing it is an operator call with the same LAN-resolution prerequisite as the six "
|
||||
"internal services (Pi-hole must answer first).",
|
||||
"publishes grafana.${BASE_DOMAIN} -> the tunnel (proxied CNAME). An admin dashboard "
|
||||
"behind Keycloak; it is not a catalog service, so withdrawing it is an operator call per "
|
||||
"name (it stayed public in the 2026-11-08 tunnel decision).",
|
||||
"platform/headlamp/httproute.yaml:headlamp/headlamp":
|
||||
"publishes headlamp.${BASE_DOMAIN} -> the Gateway VIP. A Kubernetes web UI with native "
|
||||
"Keycloak OIDC; same adjudication as grafana.",
|
||||
"publishes NOTHING since 2026-11-08: the route carries `controller: none` (operator "
|
||||
"decision: Headlamp is LAN/VPN-only) and the tunnel 404s the name. A Kubernetes web UI "
|
||||
"with native Keycloak OIDC; listed because it is a hand-authored route, and pinned "
|
||||
"disowned by (9d).",
|
||||
"platform/vuln-scanner/httproute.yaml:vuln-scanner/vuln-scanner":
|
||||
"publishes NOTHING: the route carries `controller: none` (docs/17: the CVE page is internal "
|
||||
"only) and the tunnel 404s the name. Hand-authored because the scanner is not a catalog "
|
||||
"service (its cluster-wide read role cannot be expressed in a ServiceDefinition, ADR-0027).",
|
||||
"platform/homepage/httproute.yaml:homepage/homepage":
|
||||
"publishes the APEX ${BASE_DOMAIN} itself -> the Gateway VIP, via the https-apex "
|
||||
"publishes the APEX ${BASE_DOMAIN} itself -> the tunnel (proxied CNAME), via the https-apex "
|
||||
"listener. The apex is not a per-service name and cannot be withdrawn by an exposure "
|
||||
"declaration.",
|
||||
"runbooks/serving/httproute.yaml:runbooks/runbooks":
|
||||
@@ -168,16 +201,16 @@ KNOWN_NON_CATALOG_PUBLISHERS: dict[str, str] = {
|
||||
"entry names the homepage route). Deliberately un-authenticated — see the manifest's own "
|
||||
"header for why gating runbooks on Keycloak is a loop with no exit.",
|
||||
"platform/crafty/httproute.yaml:crafty/crafty":
|
||||
"publishes crafty.${BASE_DOMAIN} -> the Gateway VIP (the Minecraft admin UI, gated by an "
|
||||
"publishes crafty.${BASE_DOMAIN} -> the tunnel (proxied CNAME) (the Minecraft admin UI, gated by an "
|
||||
"oauth2-proxy sidecar). Platform component, no exposure field.",
|
||||
"platform/crafty/service-l4.yaml:crafty/crafty-l4-tcp":
|
||||
"the `service` SOURCE, not a route: publishes crafty-mc.${BASE_DOMAIN} -> "
|
||||
"${CRAFTY_L4_VIP} (192.168.2.22) for the raw Minecraft TCP/UDP game ports the HTTPS "
|
||||
"Gateway cannot express. A game server is the one case where a public name may be "
|
||||
"genuinely wanted; the address it answers with is still RFC1918.",
|
||||
"genuinely wanted; the address it answers with is still RFC1918, DNS-only (9b).",
|
||||
"apps/vms/unifi/50-service-l4.yaml:unifi/unifi-l4-tcp":
|
||||
"the `service` source: publishes unifi.${BASE_DOMAIN} -> ${UNIFI_L4_VIP} "
|
||||
"(192.168.2.21) for the UniFi controller VM's inform/STUN/adoption ports. Hand-authored "
|
||||
"(192.168.2.21), DNS-only (9b), for the UniFi controller VM's inform/STUN/adoption ports. Hand-authored "
|
||||
"under apps/vms (a VM, not a rendered container service); the catalog's only VM entry is "
|
||||
"vm-demo, which declares no ingressRoutes and publishes nothing.",
|
||||
}
|
||||
@@ -231,9 +264,7 @@ for sub in SCAN_DIRS:
|
||||
ann = md.get("annotations") or {}
|
||||
if kind == "HTTPRoute":
|
||||
routes.append((rel, d))
|
||||
elif kind == "Service" and any(
|
||||
a.startswith("external-dns.alpha.kubernetes.io/") for a in ann
|
||||
):
|
||||
elif kind == "Service" and any(EXTERNAL_DNS_KEY_RE.match(a) for a in ann):
|
||||
dns_services.append((rel, d))
|
||||
|
||||
check(len(routes) > 0, "inventory: no HTTPRoute found under apps/ platform/ runbooks/ — parse "
|
||||
@@ -444,6 +475,174 @@ check(
|
||||
"and the whole-repo-invariants-wired meta-test fails the gate on a hand-written stage.",
|
||||
)
|
||||
|
||||
# =============================================================================================
|
||||
# (8) THE RUNNING CONTROLLER READS THE PREFIX THIS TREE WRITES.
|
||||
# (a) the external-dns HelmRelease sets `values.annotationPrefix` EXPLICITLY (an upstream
|
||||
# default is exactly what moved under us) and it equals ANNOTATION_PREFIX;
|
||||
# (b) it does not ALSO enable `--enable-legacy-annotation-prefix` (dual-read hides a
|
||||
# half-migration; upstream says the flag will be removed);
|
||||
# (c) every external-dns-looking annotation key on any object under apps/ platform/ runbooks/
|
||||
# uses ANNOTATION_PREFIX — a key under another prefix is silently ignored at runtime;
|
||||
# (d) catalog/render.py emits no external-dns key under another prefix.
|
||||
# Migrating to the GA prefix = change ANNOTATION_PREFIX, the HelmRelease value, the renderer
|
||||
# and every manifest in ONE commit; this assertion fails on any partial state.
|
||||
# =============================================================================================
|
||||
HR_PATH = ROOT / "platform/externaldns/helmrelease.yaml"
|
||||
hr = next((d for d in docs_of(HR_PATH) if d.get("kind") == "HelmRelease"
|
||||
and (d.get("metadata") or {}).get("name") == "external-dns"), None)
|
||||
check(hr is not None, f"(8) {HR_PATH.relative_to(ROOT)} holds no HelmRelease `external-dns` — "
|
||||
f"cannot prove which annotation prefix the controller reads")
|
||||
values = ((hr or {}).get("spec") or {}).get("values") or {}
|
||||
hr_prefix = values.get("annotationPrefix")
|
||||
check(
|
||||
hr_prefix == ANNOTATION_PREFIX,
|
||||
f"(8a) external-dns HelmRelease `values.annotationPrefix` is {hr_prefix!r}, but every "
|
||||
f"external-dns annotation in this tree uses {ANNOTATION_PREFIX!r}. A mismatch (or relying on "
|
||||
f"the upstream default, which changed in v0.22.0) makes the controller IGNORE `controller: "
|
||||
f"none`, the Gateway `target` and Service `hostname` — internal names go public, public names "
|
||||
f"answer RFC1918 (incident 2026-11-07).",
|
||||
)
|
||||
extra = values.get("extraArgs") or []
|
||||
extra_s = " ".join(f"{k}={v}" for k, v in extra.items()) if isinstance(extra, dict) \
|
||||
else " ".join(map(str, extra))
|
||||
check("enable-legacy-annotation-prefix" not in extra_s and "annotation-prefix" not in extra_s,
|
||||
"(8b) external-dns extraArgs must not set --annotation-prefix or "
|
||||
"--enable-legacy-annotation-prefix; the prefix is declared ONLY via values.annotationPrefix "
|
||||
"so this check can see it")
|
||||
|
||||
foreign: list[str] = []
|
||||
seen_keys = 0
|
||||
for sub in SCAN_DIRS:
|
||||
base = ROOT / sub
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in sorted(base.rglob("*.yaml")):
|
||||
rel = p.relative_to(ROOT).as_posix()
|
||||
for d in docs_of(p):
|
||||
for k in annotations_of(d):
|
||||
if isinstance(k, str) and EXTERNAL_DNS_KEY_RE.match(k):
|
||||
seen_keys += 1
|
||||
if not k.startswith(ANNOTATION_PREFIX):
|
||||
foreign.append(f"{rel}:{'/'.join(ident(d))} {k}")
|
||||
check(seen_keys > 0, "(8c) no external-dns annotation found anywhere — parse contract broken "
|
||||
"(the Gateway target alone guarantees at least one)")
|
||||
check(not foreign,
|
||||
f"(8c) external-dns annotations under a prefix the controller does NOT read "
|
||||
f"({ANNOTATION_PREFIX!r} is configured): {foreign}")
|
||||
render_src = read(CATALOG / "render.py")
|
||||
bad_render = sorted({m for m in re.findall(r"external-dns(?:\.[a-z0-9-]+)*\.kubernetes\.io/",
|
||||
render_src) if m != ANNOTATION_PREFIX})
|
||||
check(not bad_render,
|
||||
f"(8d) catalog/render.py emits external-dns keys under {bad_render}, but the controller "
|
||||
f"reads {ANNOTATION_PREFIX!r}")
|
||||
|
||||
# =============================================================================================
|
||||
# (9) THE CLOUDFLARE TUNNEL WIRING (2026-11-08). The public names are PROXIED CNAMEs to the tunnel.
|
||||
# =============================================================================================
|
||||
CF_CONFIG = ROOT / "platform/cloudflared/config.yaml"
|
||||
GATEWAY_FILE = ROOT / "platform/gateway/gateway.yaml"
|
||||
TARGET_KEY = ANNOTATION_PREFIX + "target"
|
||||
PROXIED_KEY = ANNOTATION_PREFIX + "cloudflare-proxied"
|
||||
UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
|
||||
# The L4 names the tunnel 404s that are NONETHELESS published — as DNS-only A records from the
|
||||
# `service` source, at their own LB VIP, because they are not HTTP and never touch the tunnel
|
||||
# (operator decision 2026-11-08). Shrink-only, and must agree with the tunnel's 404 list (9c).
|
||||
L4_DNS_ONLY_NAMES = {"unifi.${BASE_DOMAIN}", "crafty-mc.${BASE_DOMAIN}"}
|
||||
# The operator's LAN/VPN-only HTTP names (2026-11-08). Pinned here as well as in the tunnel config, so
|
||||
# deleting one from config.yaml cannot quietly also release it from (9c).
|
||||
OPERATOR_LAN_ONLY_NAMES = {"plex.${BASE_DOMAIN}", "headlamp.${BASE_DOMAIN}"}
|
||||
|
||||
cf = next(iter(docs_of(CF_CONFIG)), None)
|
||||
check(cf is not None and isinstance(cf.get("ingress"), list),
|
||||
f"(9) {CF_CONFIG.relative_to(ROOT)} unreadable or has no `ingress` list — cannot prove what "
|
||||
f"the tunnel refuses or which tunnel the Gateway must target")
|
||||
cf = cf or {}
|
||||
tunnel = str(cf.get("tunnel") or "")
|
||||
check(bool(UUID_RE.match(tunnel)),
|
||||
f"(9a) platform/cloudflared/config.yaml `tunnel:` is {tunnel!r}, not a literal tunnel UUID — "
|
||||
f"the Gateway target is derived from it")
|
||||
gw = next((d for d in docs_of(GATEWAY_FILE) if d.get("kind") == "Gateway"
|
||||
and ident(d) == ("gateway", "shared")), None)
|
||||
check(gw is not None, f"(9a) {GATEWAY_FILE.relative_to(ROOT)} holds no Gateway gateway/shared")
|
||||
want_target = f"{tunnel}.cfargotunnel.com"
|
||||
got_target = annotations_of(gw or {}).get(TARGET_KEY)
|
||||
check(got_target == want_target,
|
||||
f"(9a) Gateway gateway/shared `{TARGET_KEY}` is {got_target!r}; it must be EXACTLY "
|
||||
f"{want_target!r} (the tunnel in platform/cloudflared/config.yaml). Any other value publishes "
|
||||
f"every public name at the wrong place — a dead cfargotunnel.com name (Cloudflare 1016), or "
|
||||
f"an origin address of ours.")
|
||||
check("--cloudflare-proxied" in [str(a) for a in (extra if isinstance(extra, list) else [])]
|
||||
or (isinstance(extra, dict) and str(extra.get("cloudflare-proxied", "")).lower() == "true"),
|
||||
"(9a) external-dns HelmRelease extraArgs must include `--cloudflare-proxied`: a DNS-only "
|
||||
"CNAME to <uuid>.cfargotunnel.com resolves to nothing routable, so every public name is dead")
|
||||
|
||||
for rel, d in dns_services:
|
||||
if is_disowned(d):
|
||||
continue
|
||||
if HOSTNAME_KEY not in annotations_of(d):
|
||||
continue
|
||||
v = annotations_of(d).get(PROXIED_KEY)
|
||||
check(str(v).lower() == "false",
|
||||
f"(9b) {rel} Service {'/'.join(ident(d))} publishes via the `service` source but carries "
|
||||
f"`{PROXIED_KEY}: {v!r}`, not \"false\". With --cloudflare-proxied global it would become "
|
||||
f"an orange-clouded A record, which cannot carry L4 traffic and hides nothing (the LB VIP "
|
||||
f"is RFC1918). Every `service`-source publisher must opt out explicitly.")
|
||||
for rel, d in routes:
|
||||
v = annotations_of(d).get(PROXIED_KEY)
|
||||
check(v is None or str(v).lower() == "true",
|
||||
f"(9b) {rel} HTTPRoute {'/'.join(ident(d))} sets `{PROXIED_KEY}: {v!r}`. A gateway-sourced "
|
||||
f"record is a CNAME to the tunnel; DNS-only, it is dead. Disown the route instead "
|
||||
f"(`{CONTROLLER_KEY}: none`) if the name should not be public.")
|
||||
|
||||
refused = {str(r.get("hostname")) for r in cf.get("ingress") or []
|
||||
if isinstance(r, dict) and r.get("hostname") and r.get("service") == "http_status:404"}
|
||||
check(len(refused) > 0, "(9c) the tunnel config 404s no hostname — parse contract broken (the "
|
||||
"operator's LAN/VPN-only list is known to be non-empty)")
|
||||
published_l4: set[str] = set()
|
||||
for rel, d in routes:
|
||||
hosts = {str(h) for h in (d.get("spec") or {}).get("hostnames") or []}
|
||||
leaked = sorted(hosts & refused)
|
||||
if is_disowned(d):
|
||||
unrefused = sorted(hosts - refused)
|
||||
check(not unrefused,
|
||||
f"(9c) {rel} HTTPRoute {'/'.join(ident(d))} is disowned (no public record) but the "
|
||||
f"tunnel does not 404 {unrefused}: add them to platform/cloudflared/config.yaml above "
|
||||
f"the wildcard, or the two halves of \"LAN/VPN-only\" have drifted apart.")
|
||||
continue
|
||||
check(not leaked,
|
||||
f"(9c) {rel} HTTPRoute {'/'.join(ident(d))} PUBLISHES {leaked}, which the tunnel maps to "
|
||||
f"http_status:404 (LAN/VPN-only). The name would be a public proxied record that serves "
|
||||
f"only 404s — disown the route (`{CONTROLLER_KEY}: none`) or adjudicate the name.")
|
||||
for rel, d in dns_services:
|
||||
if is_disowned(d):
|
||||
continue
|
||||
hosts = {h.strip() for h in str(annotations_of(d).get(HOSTNAME_KEY) or "").split(",")
|
||||
if h.strip()}
|
||||
for h in sorted(hosts & refused):
|
||||
published_l4.add(h)
|
||||
check(h in L4_DNS_ONLY_NAMES,
|
||||
f"(9c) {rel} Service {'/'.join(ident(d))} publishes {h}, which the tunnel 404s, and it "
|
||||
f"is not a declared L4 DNS-only name {sorted(L4_DNS_ONLY_NAMES)}")
|
||||
check(L4_DNS_ONLY_NAMES <= refused,
|
||||
f"(9c) L4 DNS-only names {sorted(L4_DNS_ONLY_NAMES - refused)} are not 404'd by the tunnel; a "
|
||||
f"stray CNAME to the tunnel for them would hit the Gateway's catch-all instead of a 404")
|
||||
check(published_l4 == L4_DNS_ONLY_NAMES,
|
||||
f"(9c) L4_DNS_ONLY_NAMES is shrink-only: {sorted(L4_DNS_ONLY_NAMES - published_l4)} no longer "
|
||||
f"published by any `service`-source publisher — delete them from the list")
|
||||
|
||||
check(OPERATOR_LAN_ONLY_NAMES <= refused,
|
||||
f"(9d) operator LAN/VPN-only names {sorted(OPERATOR_LAN_ONLY_NAMES - refused)} are missing "
|
||||
f"from the tunnel's http_status:404 rules (platform/cloudflared/config.yaml)")
|
||||
for name in sorted(OPERATOR_LAN_ONLY_NAMES):
|
||||
naming = [(rel, d) for rel, d in routes
|
||||
if name in {str(h) for h in (d.get("spec") or {}).get("hostnames") or []}]
|
||||
check(len(naming) > 0, f"(9d) no HTTPRoute names {name} — stale OPERATOR_LAN_ONLY_NAMES entry")
|
||||
for rel, d in naming:
|
||||
check(is_disowned(d),
|
||||
f"(9d) {rel} HTTPRoute {'/'.join(ident(d))} names LAN/VPN-only {name} but is not "
|
||||
f"disowned, so external-dns publishes it publicly (operator decision 2026-11-08)")
|
||||
|
||||
# =============================================================================================
|
||||
if FAILURES:
|
||||
for f in FAILURES:
|
||||
|
||||
@@ -25,15 +25,19 @@
|
||||
#
|
||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||
#
|
||||
# Runtime is ~20s (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster).
|
||||
# Runtime is ~11s serial (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||
# verdicts are independent of the pool size.
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -136,43 +140,99 @@ MUTATIONS: list[tuple[str, str, str, str]] = [
|
||||
]
|
||||
|
||||
|
||||
def mutation_jobs() -> int:
|
||||
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||
|
||||
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||
"""
|
||||
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||
if raw:
|
||||
try:
|
||||
n = int(raw)
|
||||
except ValueError:
|
||||
n = 0
|
||||
if n < 1:
|
||||
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||
return n
|
||||
try:
|
||||
cpus = len(os.sched_getaffinity(0))
|
||||
except (AttributeError, OSError):
|
||||
cpus = os.cpu_count() or 1
|
||||
return max(1, min(8, cpus))
|
||||
|
||||
|
||||
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(mutated_src)
|
||||
path = fh.name
|
||||
try:
|
||||
return subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=240,
|
||||
stdin=subprocess.DEVNULL)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||
return 2
|
||||
src = Path(sys.argv[1]).read_text()
|
||||
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
try:
|
||||
jobs = mutation_jobs()
|
||||
except ValueError as e:
|
||||
print("mutations.py: %s" % e, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||
for name, why, old, new in MUTATIONS:
|
||||
old, new = dedent4(old), dedent4(new)
|
||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||
# moved and this mutation silently stopped proving anything.
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
planned.append((name, why,
|
||||
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
"code moved and this mutation no longer proves anything. "
|
||||
"Re-anchor it against the current source." % (name, n))
|
||||
"Re-anchor it against the current source." % (name, n), None))
|
||||
continue
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(src.replace(old, new, 1))
|
||||
path = fh.name
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=240)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-42s %s" % (name, how))
|
||||
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||
|
||||
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||
try:
|
||||
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||
for (_n, _w, _a, mutated) in planned]
|
||||
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||
if anchor_problem is not None:
|
||||
problems.append(anchor_problem)
|
||||
continue
|
||||
proc = fut.result()
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-42s %s" % (name, how))
|
||||
finally:
|
||||
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||
# serially; queued mutations are cancelled rather than run for nothing.
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
print()
|
||||
if problems:
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the passkey configuration is derived from one declaration, continuously
|
||||
# reconciled, and the browser login flow converges BLUE/GREEN: the BOUND flow is
|
||||
# unwritable, and binding a new one is git's explicit decision (bind-browser-flow)
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# R-IDP-03 (identity config is declarative) and R-IDP-04 (MFA available) are both
|
||||
# implemented through a mechanism that FAILS OPEN: the KeycloakRealmImport is a ONE-SHOT.
|
||||
# Measured on prod 2026-10-18, hours after the passkey declaration merged:
|
||||
#
|
||||
# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}'
|
||||
# Done=True Started=False HasErrors=False
|
||||
#
|
||||
# Once it rests at Done=True the operator never re-runs it, so the WebAuthn passwordless
|
||||
# policy and the restructured `browser-mfa` flow that keycloak-realm-import.yaml declares
|
||||
# are INERT — in git, applied by Flux, reported Ready, reaching the realm never. Merging
|
||||
# them changed nothing about how anyone logs in, and nothing anywhere said so.
|
||||
# Declarative config that silently does not apply is not declarative config.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN the reconciler's desired state is GENERATED from the realm import (so the RP ID —
|
||||
# which is baked into every passkey at enrolment and can NEVER be re-keyed — is
|
||||
# declared exactly once); both files are rendered by the SAME Flux Kustomization, so
|
||||
# their ${BASE_DOMAIN} token cannot resolve to two values; the renderer's exported
|
||||
# policy keys and the reconciler's writable keys are the SAME tuple; the realm
|
||||
# import declares NO requiredActions block (26.0.6 treats it as REPLACE, which would
|
||||
# delete CONFIGURE_TOTP and lock out every platform-admin); the enrolment action is
|
||||
# enabled but NOT a defaultAction, so enrolment stays opt-in; the reconciler carries
|
||||
# no ServiceAccount token, mounts only what it needs, passes no mode flag, and
|
||||
# contains no dollar sign; the endpoint allow-list contains NO delete, NO
|
||||
# single-execution or config-edit write, and NO required-action write for any alias
|
||||
# other than the one named literally; the protected-subtree guard exists, is wired
|
||||
# into the single request choke point and is fail-closed; the SIBLING client-secret
|
||||
# reconciler's allow-list is NOT widened; and the reconciler's own `--selftest`
|
||||
# (extracted from the shipped ConfigMap and executed) proves compare-then-write,
|
||||
# idempotence, strict read-modify-write, the RpEntityName gate, the bound flow and
|
||||
# its sub-flows unwritable, the git bind switch and a rollback that sticks, the
|
||||
# conditional-OTP regression and fail-closed — with mutations.py proving that
|
||||
# selftest can FAIL.
|
||||
#
|
||||
# WHY THE FLOW IS BUILT BLUE/GREEN AND BOUND ONLY ON GIT'S SAY-SO, asserted here rather
|
||||
# than only argued: converging a five-flow, ten-execution restructure needs
|
||||
# create-subflow, create-execution, set-requirement and bind-config as SEPARATE admin API
|
||||
# calls with no transaction. Doing that to the BOUND flow means a run that dies between
|
||||
# them leaves a HALF-BUILT FLOW AS THE LOGIN PATH for Grafana, Gitea, Nextcloud, Headlamp,
|
||||
# Semaphore, PiHole, Crafty, the *arr stack AND kubectl — this repo has had a total SSO
|
||||
# outage from a realm change (2026-09-06), and the conditional-OTP sub-flow once 403'd
|
||||
# every platform-admin. So every multi-call write lands on a NEW, unused candidate flow,
|
||||
# which is verified by readback before anything is bound. Binding it is then ONE atomic
|
||||
# PUT of browserFlow, made only when passkey-desired-state.yaml sets
|
||||
# bind-browser-flow: "true" (default "false"): an untested login flow never goes live on
|
||||
# a schedule tick, and with the switch off an operator's rollback is never undone.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (git + PyYAML +
|
||||
# stdlib); conformance/run.sh --layer l0 and ci/l0.sh run it directly, and this Chainsaw
|
||||
# Test wraps the SAME script.
|
||||
#
|
||||
# NOT proven here: that the LIVE realm's policy, enrolment action or browser flow match
|
||||
# git (that is cluster state — the reconciler's `--audit` mode is the read-only probe for
|
||||
# it, and NO L2 test is added by this change, so the live realm is UNVERIFIED by CI);
|
||||
# that a passkey ceremony actually completes or that the ALTERNATIVE fork degrades to the
|
||||
# password form (needs a real browser against a real realm); that the five flow-building
|
||||
# admin-API calls behave against a live Keycloak the way the 26.0.6 source says they do
|
||||
# (every proof here is against an in-process stub); or that the CronJob's schedule fires.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: keycloak-passkey-reconciler
|
||||
labels:
|
||||
conformance.home.k3s/R-IDP-03: "true"
|
||||
conformance.home.k3s/R-IDP-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-IDP-03, R-IDP-04"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE, for the same reason the sibling keycloak-client-secret-reconciler
|
||||
# check declares it: this invariant spans platform/ (the realm import, the reconciler,
|
||||
# the generated desired state), platform-config/ (which Flux Kustomization renders
|
||||
# them, and therefore which substitution pass they share), ci/ (the renderer) and
|
||||
# clusters/ (that both are reachable from both entrypoints). A change that breaks it
|
||||
# need not attribute R-IDP-03/04 at all, and the requirement-scoped selection would
|
||||
# then filter this check out of exactly the change that introduces the bug — which is
|
||||
# how 12 unusable prod OIDC clients sat behind an entirely green cluster on
|
||||
# 2026-09-06.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
# Not safety-critical in the harness's sense (it mutates nothing and needs no
|
||||
# cluster), but it guards a component that holds WRITE access to the identity
|
||||
# provider's realm settings.
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
The realm's passkey configuration — policy, enrolment action AND browser login
|
||||
flow — is declared once in the realm import, derived from it by a gated renderer,
|
||||
and continuously reconciled; the flow converges blue/green, so a one-shot import
|
||||
that never re-runs can neither leave the configuration inert nor let a reconciler
|
||||
half-rewrite the login path of the whole platform.
|
||||
spec:
|
||||
steps:
|
||||
- name: audit-passkey-declaration-renderer-and-reconciler
|
||||
description: >-
|
||||
Assert the desired state is derived from the realm import and shares its
|
||||
substitution pass, that the reconciler cannot write the bound flow or delete
|
||||
anything and binds only on git's bind-browser-flow, that the sibling reconciler
|
||||
is unwidened, and run the reconciler's own selftest plus the mutation suite that
|
||||
proves it can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,754 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — keycloak-passkey-reconciler
|
||||
# Serves: R-IDP-03 ("Identity config (realms, clients, groups, roles) is declarative."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
# R-IDP-04 ("IdP secrets SOPS-encrypted; MFA available."
|
||||
# — spec docs/15-identity-and-access.md)
|
||||
#
|
||||
# WHY THIS EXISTS — THE REALM IMPORT IS A ONE-SHOT THAT FAILS OPEN
|
||||
# ---------------------------------------------------------------
|
||||
# platform/keycloak/keycloak-realm-import.yaml declares the WebAuthn PASSWORDLESS
|
||||
# policy and a restructured `browser-mfa` browser flow. Measured on prod 2026-10-18,
|
||||
# hours after that declaration merged:
|
||||
#
|
||||
# kubectl -n keycloak get keycloakrealmimport homelab -o jsonpath='{.status}'
|
||||
# Done=True Started=False HasErrors=False
|
||||
#
|
||||
# A KeycloakRealmImport is a ONE-SHOT. Once it rests at Done=True the operator never
|
||||
# re-runs it, so every realm-level field and every flow declared after the FIRST
|
||||
# import is inert — in git, applied by Flux, reported Ready, reaching the realm
|
||||
# never. The same shape as the 2026-09-06 client-secret incident (12 prod clients
|
||||
# holding the literal `(env:...)` placeholder behind an entirely green cluster), and
|
||||
# the same fix: a level-triggered reconciler. This check is the static half.
|
||||
#
|
||||
# THE RECONCILER HOLDS WRITE ACCESS TO THE IDENTITY PROVIDER. That is the reason this
|
||||
# check is unusually thorough about its BLAST RADIUS rather than only its behaviour.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) THE DESIRED STATE IS DERIVED, NOT DECLARED TWICE: the reconciler's input
|
||||
# (platform/keycloak/passkey-desired-state.yaml) is GENERATED from the realm
|
||||
# import by ci/render-keycloak-passkey-policy.py, and this check RUNS the
|
||||
# renderer with --check, so a hand edit that makes the two disagree is
|
||||
# unmergeable. The RP ID is the sharpest case: it is baked into every passkey
|
||||
# at enrolment and can NEVER be re-keyed, so two declarations that can skew
|
||||
# about the domain is an unrecoverable-credential bug, not a cosmetic one.
|
||||
# (2) BOTH FILES ARE RENDERED BY THE SAME FLUX KUSTOMIZATION.
|
||||
# postBuild.substituteFrom is per-Kustomization, so a split would let the
|
||||
# ${BASE_DOMAIN} token in the realm import and the one in the desired state
|
||||
# resolve to two different values — and an RP ID mismatch invalidates every
|
||||
# enrolled passkey with no error the user can act on.
|
||||
# (3) THE DESIRED STATE USES NO SUBSTITUTION TOKEN THE REALM IMPORT DOES NOT, and
|
||||
# declares no realm key outside the reconciler's write surface.
|
||||
# (4) THE TWO WRITE SURFACES AGREE: the renderer's exported policy keys and the
|
||||
# reconciler's POLICY_FIELDS are the SAME tuple. If they drifted, the renderer
|
||||
# could emit a key the reconciler refuses (fail-closed forever) or the
|
||||
# reconciler could accept a key the renderer never emits (dead surface).
|
||||
# (5) THE RECONCILER MANIFEST IS SAFE AND WIRED: reconciled from BOTH cluster
|
||||
# entrypoints, mounts exactly the ConfigMaps its code defaults to, carries NO
|
||||
# ServiceAccount token, and contains no dollar sign (it ships inside a
|
||||
# SUBSTITUTED Flux path, where an unescaped dollar-brace expansion is
|
||||
# rewritten before any container runs).
|
||||
# (6) THE CURRENTLY BOUND FLOW IS STRUCTURALLY UNWRITABLE — asserted against the
|
||||
# SOURCE, not the comments. The flow converges BLUE/GREEN (build under a new
|
||||
# alias, verify, flip browserFlow in one atomic PUT), so flow-create and
|
||||
# execution-add endpoints ARE on the allow-list and their presence is not the
|
||||
# hazard. What is asserted instead: nothing is DELETABLE (the superseded flow
|
||||
# is the rollback target); the execution- and config-id-scoped write endpoints
|
||||
# stay off the list because they carry no flow alias and so cannot be screened;
|
||||
# no required-action write reaches any alias but the one named literally; and
|
||||
# the guard the allow-list CANNOT express exists in the source and is wired
|
||||
# into the single request choke point, FAIL-CLOSED until the live binding has
|
||||
# been read. That last one is what the whole design rests on: a reconciler that
|
||||
# half-rewrites the BOUND browser flow is a total SSO outage on a loop, and
|
||||
# this repo has had one (2026-09-06).
|
||||
# (6b) THE BIND IS GIT'S DECISION: the desired-state ConfigMap carries
|
||||
# bind-browser-flow as exactly "true" or "false", the script reads it from the
|
||||
# SAME mount, and the CronJob passes no flag that could widen it. With "false"
|
||||
# (the default) the reconciler builds and verifies but never PUTs browserFlow,
|
||||
# so an operator's rollback sticks; the selftest proves that end to end.
|
||||
# (7) THE SIBLING RECONCILER IS UNTOUCHED: client-secret-reconciler.yaml's
|
||||
# allow-list is NOT widened by this work. Its --selftest asserts it refuses
|
||||
# /users, /groups, /roles, /authentication and the whole-realm endpoint, and
|
||||
# its guarantee that prod's real user and the media-users / platform-admins /
|
||||
# viewers groups survive every run must stay LITERALLY true.
|
||||
# (8) THE RECONCILER'S BEHAVIOUR IS PROVEN, NOT ASSERTED: this check EXTRACTS the
|
||||
# script from the ConfigMap and executes its `--selftest`, which drives the
|
||||
# real code paths against an in-process stub Keycloak that MODELS the three
|
||||
# Keycloak 26.0.6 behaviours that bite — the RpEntityName gate on the
|
||||
# passwordless policy, the all-or-nothing policy rebuild that defaults every
|
||||
# omitted key, and the required-action representation replace.
|
||||
# (8b) AND THAT PROOF IS FALSIFIABLE: mutations.py injects plausible regressions
|
||||
# one at a time and requires --selftest to CATCH every one. A passing suite
|
||||
# proves nothing if it cannot fail, and "reported success while the realm was
|
||||
# broken" is the exact failure this component exists to end.
|
||||
# AND (9) THIS CHECK IS LOAD-BEARING: it DECLARES
|
||||
# `conformance.home.k3s/scope: whole-repo` so ci/l0.sh runs it unconditionally
|
||||
# via --include-scope, ci/gate.sh blocks the merge when the l0 stage fails, and
|
||||
# ci/touched-requirements.sh maps platform/keycloak/ to these requirement IDs.
|
||||
# An unreferenced test is precisely the "looks applied and isn't" failure this
|
||||
# repo keeps hitting.
|
||||
#
|
||||
# NOT proven here (named honestly, in the style of the sibling L0 checks):
|
||||
# - that the LIVE realm's policy, enrolment action or browser flow match git. That is
|
||||
# cluster state. The reconciler's own `--audit` mode is the read-only probe for it
|
||||
# and is what an L2 test should run; no such L2 test is added by this change, so the
|
||||
# live realm is UNVERIFIED by CI.
|
||||
# - that the declared browser flow actually WORKS as a login path. Nothing offline can
|
||||
# prove that a passkey ceremony completes, that the ALTERNATIVE fork degrades to the
|
||||
# password form, or that a platform-admin is still asked for TOTP. Those need a real
|
||||
# browser against a real realm.
|
||||
# - that the reconciler's first real run against prod will succeed. The flow is built
|
||||
# through five admin-API endpoints whose request shapes were read from the 26.0.6
|
||||
# source but never exercised against a live Keycloak.
|
||||
# - that the admin credential decrypts to a usable password. Key PRESENCE is checkable
|
||||
# offline; the plaintext is not, and must not be.
|
||||
# - that the CronJob's image can be pulled or its schedule fires.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
IMPORT_FILE="platform/keycloak/keycloak-realm-import.yaml"
|
||||
RECON_FILE="platform/keycloak/passkey-reconciler.yaml"
|
||||
STATE_FILE="platform/keycloak/passkey-desired-state.yaml"
|
||||
RENDERER="ci/render-keycloak-passkey-policy.py"
|
||||
SIBLING_FILE="platform/keycloak/client-secret-reconciler.yaml"
|
||||
|
||||
# Honest SKIP (never green) if the thing under test is absent entirely.
|
||||
for f in "$IMPORT_FILE" "$RECON_FILE" "$STATE_FILE" "$RENDERER"; do
|
||||
if [ ! -f "${ROOT}/${f}" ]; then
|
||||
echo "check SKIPPED: ${f} absent — nothing to audit" >&2
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
"""Static audit of the Keycloak passkey declaration, the renderer that derives the
|
||||
reconciler's desired state from it, and the reconciler's blast radius. PyYAML parses the
|
||||
manifests; lib/status.py is IMPORTED for the Flux graph walk rather than forked, so there
|
||||
is one definition of "what a Flux path renders" in the harness."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
# This body is fed to `python3 -` on stdin, so there is no __file__ to derive from; the
|
||||
# test directory is named relative to the repo root the wrapper passes in.
|
||||
DIR_PATH = (ROOT / "conformance" / "tests" / "l0-static"
|
||||
/ "keycloak-passkey-reconciler")
|
||||
sys.path.insert(0, str(ROOT / "conformance" / "lib"))
|
||||
import status as st # noqa: E402
|
||||
|
||||
IMPORT_FILE = "platform/keycloak/keycloak-realm-import.yaml"
|
||||
RECON_FILE = "platform/keycloak/passkey-reconciler.yaml"
|
||||
STATE_FILE = "platform/keycloak/passkey-desired-state.yaml"
|
||||
RENDERER = "ci/render-keycloak-passkey-policy.py"
|
||||
SIBLING_FILE = "platform/keycloak/client-secret-reconciler.yaml"
|
||||
STATE_CM_NAME = "keycloak-passkey-desired-state"
|
||||
STATE_DATA_KEY = "passkey-desired-state.json"
|
||||
BIND_KEY = "bind-browser-flow"
|
||||
SCRIPT_KEY = "reconcile-keycloak-passkeys.py"
|
||||
ENROL_ACTION = "webauthn-register-passwordless"
|
||||
DOLLAR = chr(36)
|
||||
|
||||
failures: list[str] = []
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
if not cond:
|
||||
failures.append(msg)
|
||||
|
||||
|
||||
def load(rel):
|
||||
return list(yaml.safe_load_all((ROOT / rel).read_text()))
|
||||
|
||||
|
||||
# ================================================== (1) the desired state is DERIVED
|
||||
proc = subprocess.run([sys.executable, str(ROOT / RENDERER), "--check"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
check(proc.returncode == 0,
|
||||
f"{STATE_FILE} has DRIFTED from {IMPORT_FILE} (ci/render-keycloak-passkey-policy"
|
||||
f".py --check failed). The reconciler would converge the realm on a passkey policy "
|
||||
f"the realm import does not declare, and audit the login flow against a tree "
|
||||
f"nobody authored. The RP ID is the sharpest case: it is baked into every passkey "
|
||||
f"at enrolment and can NEVER be re-keyed, so two declarations that skew about the "
|
||||
f"domain is an unrecoverable-credential bug.\n "
|
||||
+ "\n ".join((proc.stdout + proc.stderr).strip().splitlines()[-12:]))
|
||||
|
||||
# The realm import must actually declare the things the renderer derives from. If it
|
||||
# stops, the renderer fails -- but say so HERE too, because a SKIP-shaped failure in the
|
||||
# renderer would otherwise be the only signal.
|
||||
imp_docs = [d for d in load(IMPORT_FILE) if d]
|
||||
imp = next((d for d in imp_docs if d.get("kind") == "KeycloakRealmImport"), None)
|
||||
check(imp is not None, f"{IMPORT_FILE}: no KeycloakRealmImport document")
|
||||
realm = ((imp or {}).get("spec") or {}).get("realm") or {}
|
||||
policy_keys_in_import = sorted(k for k in realm
|
||||
if k.startswith("webAuthnPolicyPasswordless"))
|
||||
check(policy_keys_in_import,
|
||||
f"{IMPORT_FILE} declares no webAuthnPolicyPasswordless* field, so there is no "
|
||||
f"passkey policy for the reconciler to converge")
|
||||
check(realm.get("browserFlow"),
|
||||
f"{IMPORT_FILE} declares no spec.realm.browserFlow — a perfect flow that is not "
|
||||
f"BOUND changes nothing about how anyone logs in")
|
||||
|
||||
# THE requiredActions TRAP, enforced rather than only commented. The realm import's own
|
||||
# header explains it at length: Keycloak 26.0.6 treats requiredActions in a realm
|
||||
# representation as REPLACE, not merge, so declaring ONE action imports exactly that
|
||||
# action plus delete_account and skips DefaultRequiredActions.addActions entirely --
|
||||
# deleting CONFIGURE_TOTP, UPDATE_PASSWORD, VERIFY_EMAIL and seven more. Losing
|
||||
# CONFIGURE_TOTP is a hard lock-out of every platform-admin, because auth-otp-form in the
|
||||
# browser flow still demands a TOTP they could then never enrol. The comment saying "DO
|
||||
# NOT ADD ONE" is advice; this is the gate.
|
||||
check("requiredActions" not in realm,
|
||||
f"{IMPORT_FILE} declares spec.realm.requiredActions. Keycloak 26.0.6 treats that "
|
||||
f"as REPLACE, not merge: declaring any action imports exactly those plus "
|
||||
f"delete_account and NEVER runs DefaultRequiredActions.addActions, so "
|
||||
f"CONFIGURE_TOTP, UPDATE_PASSWORD, VERIFY_EMAIL and seven more are DELETED. That "
|
||||
f"is a permanent lock-out of every platform-admin, because auth-otp-form in this "
|
||||
f"realm's browser flow still demands a TOTP they could no longer enrol. The "
|
||||
f"enrolment action is already registered by default (priority 80, enabled, "
|
||||
f"defaultAction false) and the passkey reconciler converges it through the "
|
||||
f"per-alias admin API, which has no such hazard")
|
||||
|
||||
# ============================== (2)(3) the two files share one substitution pass
|
||||
state_docs = [d for d in load(STATE_FILE) if d]
|
||||
state_cm = next((d for d in state_docs if d.get("kind") == "ConfigMap"), None)
|
||||
check(state_cm is not None, f"{STATE_FILE}: no ConfigMap")
|
||||
desired = {}
|
||||
if state_cm is not None:
|
||||
check(state_cm["metadata"]["name"] == STATE_CM_NAME,
|
||||
f"{STATE_FILE}: the ConfigMap is named "
|
||||
f"{state_cm['metadata']['name']!r}, but the CronJob mounts {STATE_CM_NAME!r}")
|
||||
data = state_cm.get("data") or {}
|
||||
check(sorted(data) == sorted([STATE_DATA_KEY, BIND_KEY]),
|
||||
f"{STATE_FILE}: expected exactly the data keys {STATE_DATA_KEY!r} and "
|
||||
f"{BIND_KEY!r}, got {sorted(data)} — the reconciler's PASSKEY_STATE_FILE / "
|
||||
f"PASSKEY_BIND_FILE defaults would not resolve")
|
||||
# THE BIND SWITCH. An unquoted true/false is a YAML BOOLEAN, which a ConfigMap rejects
|
||||
# at apply time -- failing the whole platform-config Kustomization, realm import
|
||||
# included. The renderer refuses it too; asserted here so the reason is on record.
|
||||
check(data.get(BIND_KEY) in ("true", "false"),
|
||||
f"{STATE_FILE}: data.{BIND_KEY} is {data.get(BIND_KEY)!r}; it must be the QUOTED "
|
||||
f"string \"true\" or \"false\"")
|
||||
raw = data.get(STATE_DATA_KEY, "")
|
||||
try:
|
||||
desired = json.loads(raw)
|
||||
except ValueError as e:
|
||||
check(False, f"{STATE_FILE}: the {STATE_DATA_KEY} payload is not valid JSON "
|
||||
f"({e}) — the reconciler would fail closed on every run")
|
||||
|
||||
# Both files must be rendered by ONE Flux Kustomization. substituteFrom is
|
||||
# per-Kustomization, so a split lets the same ${BASE_DOMAIN} token resolve to two
|
||||
# different values -- and an RP ID mismatch invalidates every enrolled passkey silently.
|
||||
import_owners = set()
|
||||
state_owners = set()
|
||||
recon_owners = set()
|
||||
for env in ("dev", "prod"):
|
||||
entry = f"clusters/{env}"
|
||||
if not (ROOT / entry).is_dir():
|
||||
continue
|
||||
applied, _ = st.reconciled_files(ROOT, entry)
|
||||
check(RECON_FILE in applied,
|
||||
f"{RECON_FILE} is not reachable from the {entry} entrypoint — the passkey "
|
||||
f"healer would exist on one cluster only")
|
||||
check(STATE_FILE in applied,
|
||||
f"{STATE_FILE} is not reachable from the {entry} entrypoint — the reconciler "
|
||||
f"would fail closed on that cluster forever")
|
||||
check(IMPORT_FILE in applied, f"{IMPORT_FILE} is not reachable from {entry}")
|
||||
|
||||
# Which Kustomization DIRECTORY lists each file. Asserted structurally: both must be
|
||||
# listed as resources of the same kustomization.yaml.
|
||||
for kpath in ROOT.glob("*/kustomization.yaml"):
|
||||
try:
|
||||
k = yaml.safe_load(kpath.read_text()) or {}
|
||||
except Exception:
|
||||
continue
|
||||
res = [str(r) for r in (k.get("resources") or [])]
|
||||
base = kpath.parent.name
|
||||
if any(r.endswith("keycloak-realm-import.yaml") for r in res):
|
||||
import_owners.add(base)
|
||||
if any(r.endswith("passkey-desired-state.yaml") for r in res):
|
||||
state_owners.add(base)
|
||||
if any(r.endswith("passkey-reconciler.yaml") for r in res):
|
||||
recon_owners.add(base)
|
||||
check(import_owners and import_owners == state_owners,
|
||||
f"{IMPORT_FILE} is rendered by {sorted(import_owners)} but {STATE_FILE} by "
|
||||
f"{sorted(state_owners)}. They MUST be the same Flux Kustomization: "
|
||||
f"postBuild.substituteFrom is per-Kustomization, so a split would let the "
|
||||
f"{DOLLAR}{{BASE_DOMAIN}} token in each resolve to a different value. An RP ID is "
|
||||
f"baked into every passkey at enrolment and can never be re-keyed, so a domain the "
|
||||
f"two files disagree about is an unrecoverable credential, not a cosmetic drift")
|
||||
check(recon_owners == state_owners,
|
||||
f"{RECON_FILE} is rendered by {sorted(recon_owners)} but its desired state by "
|
||||
f"{sorted(state_owners)} — the reconciler could be applied without its input")
|
||||
|
||||
# (3) no substitution token the realm definition does not also use.
|
||||
import_text = (ROOT / IMPORT_FILE).read_text()
|
||||
TOKEN = re.compile(re.escape(DOLLAR) + r"\{([A-Za-z_][A-Za-z0-9_]*)")
|
||||
state_tokens = set(TOKEN.findall((ROOT / STATE_FILE).read_text()))
|
||||
import_tokens = set(TOKEN.findall(import_text))
|
||||
# The renderer's own HEADER mentions ${BASE_DOMAIN} in prose; that is the same token the
|
||||
# policy uses, so a subset check is the right assertion rather than an exact match.
|
||||
check(state_tokens <= import_tokens,
|
||||
f"{STATE_FILE} uses substitution token(s) {sorted(state_tokens - import_tokens)} "
|
||||
f"that {IMPORT_FILE} does not. A token the realm definition never uses is either a "
|
||||
f"hand edit or a renderer bug, and an UNDEFINED one renders as the EMPTY STRING "
|
||||
f"silently — which for an RP ID means every passkey ceremony fails")
|
||||
|
||||
# ======================================= (4) the two write surfaces are the SAME tuple
|
||||
recon_docs = [d for d in load(RECON_FILE) if d]
|
||||
cm = next((d for d in recon_docs if d.get("kind") == "ConfigMap"), None)
|
||||
cj = next((d for d in recon_docs if d.get("kind") == "CronJob"), None)
|
||||
check(cm is not None, f"{RECON_FILE}: no ConfigMap carrying the reconciler script")
|
||||
check(cj is not None, f"{RECON_FILE}: no CronJob running the reconciler")
|
||||
|
||||
script = ""
|
||||
if cm is not None:
|
||||
data = cm.get("data") or {}
|
||||
check(len(data) == 1,
|
||||
f"{RECON_FILE}: the reconciler ConfigMap must carry exactly one script key "
|
||||
f"(found {sorted(data)}) — a second copy would be untested by this gate")
|
||||
check(SCRIPT_KEY in data,
|
||||
f"{RECON_FILE}: expected the script under {SCRIPT_KEY!r}, got {sorted(data)}")
|
||||
script = next(iter(data.values()), "")
|
||||
check(len(script.splitlines()) > 200,
|
||||
f"{RECON_FILE}: the extracted script is implausibly short "
|
||||
f"({len(script.splitlines())} lines) — the ConfigMap shape changed and the "
|
||||
f"selftest below would prove nothing")
|
||||
# THE nextcloud TRAP. This file is rendered by a Flux Kustomization with
|
||||
# postBuild.substituteFrom, so envsubst rewrites it before any container runs.
|
||||
check(DOLLAR not in script,
|
||||
f"{RECON_FILE}: the reconciler script contains a dollar sign. It ships inside "
|
||||
f"a SUBSTITUTED Flux path, where a dollar-brace expansion is rewritten at "
|
||||
f"reconcile time (see apps/wave-1/nextcloud.yaml). Rewrite without it — Python "
|
||||
f"never needs one; chr(36) spells it where a literal is genuinely required")
|
||||
|
||||
|
||||
def tuple_literal(src, name):
|
||||
"""The value of a module-level tuple assignment, via ast. Parsing the SOURCE rather
|
||||
than importing it keeps this check offline and side-effect free, and means the thing
|
||||
asserted is the text that ships."""
|
||||
try:
|
||||
tree = ast.parse(src)
|
||||
except SyntaxError:
|
||||
return None
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign):
|
||||
for t in node.targets:
|
||||
if isinstance(t, ast.Name) and t.id == name:
|
||||
try:
|
||||
return ast.literal_eval(node.value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
renderer_src = (ROOT / RENDERER).read_text()
|
||||
|
||||
|
||||
# POLICY_FIELDS is a CONCATENATION of other tuples in both files, so literal_eval of the
|
||||
# assignment itself cannot evaluate it. Resolve it from its parts instead. Both files name
|
||||
# their parts from the same vocabulary; a part present in one and not the other simply
|
||||
# contributes nothing, which is why the comparison below is of the resolved SETS.
|
||||
def resolve_policy(src):
|
||||
parts = ("POLICY_STR_FIELDS", "POLICY_ORDERED_LIST_FIELDS",
|
||||
"POLICY_UNORDERED_LIST_FIELDS", "POLICY_LIST_FIELDS", "POLICY_INT_FIELDS",
|
||||
"POLICY_BOOL_FIELDS")
|
||||
out = []
|
||||
for p in parts:
|
||||
v = tuple_literal(src, p)
|
||||
if v:
|
||||
out.extend(v)
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
recon_keys = resolve_policy(script)
|
||||
rend_keys = resolve_policy(renderer_src)
|
||||
check(recon_keys and recon_keys == rend_keys,
|
||||
f"the reconciler's writable policy keys {recon_keys} and the renderer's exported "
|
||||
f"keys {rend_keys} DISAGREE. If the renderer emits a key the reconciler refuses, "
|
||||
f"every run fails closed forever; if the reconciler accepts a key the renderer "
|
||||
f"never emits, that is dead write surface on the identity provider")
|
||||
# Every exported key must actually be a passwordless key. The two-factor
|
||||
# webAuthnPolicy* family is a silent no-op for passkeys (the passwordless authenticator
|
||||
# reads getWebAuthnPolicyPasswordless() only), so exporting one would be a write surface
|
||||
# with no effect.
|
||||
check(all(k.startswith("webAuthnPolicyPasswordless") for k in recon_keys),
|
||||
f"the reconciler's writable keys include a non-passwordless field: "
|
||||
f"{[k for k in recon_keys if not k.startswith('webAuthnPolicyPasswordless')]}. "
|
||||
f"The two-factor webAuthnPolicy* family is never consulted by a passkey login, so "
|
||||
f"writing it is a silent no-op")
|
||||
# And the desired state must declare only those.
|
||||
if isinstance(desired.get("webAuthnPolicyPasswordless"), dict):
|
||||
extra = sorted(set(desired["webAuthnPolicyPasswordless"]) - set(recon_keys))
|
||||
check(not extra,
|
||||
f"{STATE_FILE} declares policy key(s) {extra} outside the reconciler's write "
|
||||
f"surface — it would fail closed on every run")
|
||||
# The 26.0.6 gate: the policy is applied ONLY when RpEntityName is non-empty.
|
||||
check(desired["webAuthnPolicyPasswordless"].get(
|
||||
"webAuthnPolicyPasswordlessRpEntityName"),
|
||||
f"{STATE_FILE} carries no non-empty "
|
||||
f"webAuthnPolicyPasswordlessRpEntityName. Keycloak 26.0.6 applies the "
|
||||
f"passwordless policy only when that field is set "
|
||||
f"(DefaultExportImportManager.updateRealm), so every realm PUT would be "
|
||||
f"accepted with 204 and change nothing")
|
||||
|
||||
# THE ENROLMENT POSTURE. defaultAction true would force every user to enrol a passkey at
|
||||
# next login -- a platform-wide lock-out risk, and the exact opposite of the realm
|
||||
# import's stated "additive and optional" decision. It is one boolean, so it is gated.
|
||||
ra = desired.get("requiredAction") or {}
|
||||
check(ra.get("alias") == ENROL_ACTION,
|
||||
f"{STATE_FILE}: requiredAction.alias is {ra.get('alias')!r}, expected "
|
||||
f"{ENROL_ACTION!r} — the reconciler's allow-list names that alias literally, so "
|
||||
f"any other value is unreachable and the pass would fail closed")
|
||||
check(ra.get("enabled") is True,
|
||||
f"{STATE_FILE}: requiredAction.enabled must be true, else passkey ENROLMENT is "
|
||||
f"impossible while the login flow offers passkeys")
|
||||
check(ra.get("defaultAction") is False,
|
||||
f"{STATE_FILE}: requiredAction.defaultAction is {ra.get('defaultAction')!r}, which "
|
||||
f"must be false. True forces EVERY user to enrol a passkey at next login — this "
|
||||
f"IdP gates Grafana, Gitea, Nextcloud, Headlamp, Semaphore, PiHole, Crafty, the "
|
||||
f"*arr stack AND kubectl, so a misfiring mandatory enrolment is a total platform "
|
||||
f"lock-out. The realm import's whole stated posture is additive and OPTIONAL")
|
||||
|
||||
# ========================================= (5) the reconciler manifest is safe + wired
|
||||
if cj is not None:
|
||||
pod = cj["spec"]["jobTemplate"]["spec"]["template"]["spec"]
|
||||
ctr = pod["containers"][0]
|
||||
vols = {v["name"]: v for v in pod.get("volumes", [])}
|
||||
mounts = {m["name"]: m for m in ctr.get("volumeMounts", [])}
|
||||
env = {e["name"]: e for e in ctr.get("env", [])}
|
||||
|
||||
# NO KUBERNETES API ACCESS. This is what makes the mounted-ConfigMap design strictly
|
||||
# narrower than reading the KeycloakRealmImport CR through the API: no Role, no
|
||||
# RoleBinding, no token. It is also why this change adds no **/rbac.yaml.
|
||||
check(pod.get("automountServiceAccountToken") is False,
|
||||
f"{RECON_FILE}: the reconciler must set automountServiceAccountToken: false. "
|
||||
f"It talks only to the Keycloak admin API and reads a mounted ConfigMap, so a "
|
||||
f"ServiceAccount token is pure blast radius (R-SEC-02)")
|
||||
check("serviceAccountName" not in pod,
|
||||
f"{RECON_FILE}: the reconciler must not name a ServiceAccount")
|
||||
|
||||
cm_vols = {n: v for n, v in vols.items() if "configMap" in v}
|
||||
names = {(v["configMap"] or {}).get("name") for v in cm_vols.values()}
|
||||
check(STATE_CM_NAME in names,
|
||||
f"{RECON_FILE}: the CronJob does not mount the {STATE_CM_NAME} ConfigMap, so "
|
||||
f"the reconciler would fail closed on every run. Mounted: {sorted(names)}")
|
||||
check(cm["metadata"]["name"] in names,
|
||||
f"{RECON_FILE}: the CronJob does not mount its own script ConfigMap")
|
||||
# NO SECRET IS MOUNTED AS A VOLUME. The admin credential is consumed via
|
||||
# secretKeyRef only; projecting it as a file would widen what a readable mount leaks.
|
||||
check(not [v for v in vols.values() if "secret" in v],
|
||||
f"{RECON_FILE}: the reconciler mounts a Secret as a volume. The admin "
|
||||
f"credential is referenced by secretKeyRef only; it must never be projected "
|
||||
f"as a file here")
|
||||
|
||||
# The mount paths must be the ones the code DEFAULTS to, or the container starts and
|
||||
# fails closed forever while every manifest looks correct.
|
||||
state_vol = next((n for n, v in cm_vols.items()
|
||||
if (v["configMap"] or {}).get("name") == STATE_CM_NAME), None)
|
||||
if state_vol:
|
||||
mp = (mounts.get(state_vol) or {}).get("mountPath")
|
||||
check(mp == "/etc/keycloak-passkey",
|
||||
f"{RECON_FILE}: the desired-state ConfigMap is mounted at {mp!r}, but the "
|
||||
f"script's PASSKEY_STATE_FILE default is "
|
||||
f"/etc/keycloak-passkey/{STATE_DATA_KEY}")
|
||||
check(f'"/etc/keycloak-passkey/{STATE_DATA_KEY}"' in script,
|
||||
f"{RECON_FILE}: the script's PASSKEY_STATE_FILE default no longer matches "
|
||||
f"the mount path — they are a pair and must move together")
|
||||
check((mounts.get(state_vol) or {}).get("readOnly") is True,
|
||||
f"{RECON_FILE}: the desired-state mount must be readOnly")
|
||||
check(f'"/etc/keycloak-passkey/{BIND_KEY}"' in script,
|
||||
f"{RECON_FILE}: the script's PASSKEY_BIND_FILE default is not "
|
||||
f"/etc/keycloak-passkey/{BIND_KEY}. An unresolved bind file reads as \"false\" "
|
||||
f"(safe, but git's \"true\" would then silently never take effect)")
|
||||
check(not (state_vol and (cm_vols[state_vol]["configMap"] or {}).get("items")),
|
||||
f"{RECON_FILE}: the desired-state volume selects `items`, so one of its two "
|
||||
f"keys may not be projected")
|
||||
|
||||
# The bind decision belongs to git. A container arg could only NARROW it today
|
||||
# (--no-bind), but nothing in the CronJob may carry a mode flag at all: the shipped
|
||||
# behaviour must be readable from passkey-desired-state.yaml alone.
|
||||
cmd = [str(x) for x in (ctr.get("command") or []) + (ctr.get("args") or [])]
|
||||
check(not [x for x in cmd if x.startswith("--")],
|
||||
f"{RECON_FILE}: the CronJob passes {[x for x in cmd if x.startswith('--')]} to "
|
||||
f"the reconciler. Its mode is decided by git (bind-browser-flow), not by flags")
|
||||
# Retries burn the bounded candidate-alias budget on a deterministic failure.
|
||||
check(cj["spec"]["jobTemplate"]["spec"].get("backoffLimit") == 0,
|
||||
f"{RECON_FILE}: backoffLimit must be 0 — every failure here is deterministic "
|
||||
f"within a tick, and each retried build consumes another candidate alias")
|
||||
|
||||
# The admin credential, referenced BY NAME. Never decrypted, never a literal.
|
||||
for key in ("KEYCLOAK_ADMIN_USER", "KEYCLOAK_ADMIN_PASSWORD"):
|
||||
e = env.get(key) or {}
|
||||
ref = ((e.get("valueFrom") or {}).get("secretKeyRef") or {})
|
||||
check(ref.get("name") == "keycloak-admin-credentials",
|
||||
f"{RECON_FILE}: {key} must come from the keycloak-admin-credentials "
|
||||
f"Secret by secretKeyRef, got {e!r}")
|
||||
check("value" not in e,
|
||||
f"{RECON_FILE}: {key} carries a literal `value`. A credential in a "
|
||||
f"manifest is a plaintext secret in git")
|
||||
check((env.get("KEYCLOAK_REALM") or {}).get("value") == desired.get("realm"),
|
||||
f"{RECON_FILE}: KEYCLOAK_REALM is "
|
||||
f"{(env.get('KEYCLOAK_REALM') or {}).get('value')!r} but the desired state "
|
||||
f"declares realm {desired.get('realm')!r}. The reconciler would converge the "
|
||||
f"wrong realm, or none")
|
||||
|
||||
check(ctr["securityContext"]["readOnlyRootFilesystem"] is True,
|
||||
f"{RECON_FILE}: readOnlyRootFilesystem must be true")
|
||||
check(ctr["securityContext"]["allowPrivilegeEscalation"] is False,
|
||||
f"{RECON_FILE}: allowPrivilegeEscalation must be false")
|
||||
check(pod["securityContext"].get("runAsNonRoot") is True,
|
||||
f"{RECON_FILE}: runAsNonRoot must be true")
|
||||
check(cj["spec"].get("concurrencyPolicy") == "Forbid",
|
||||
f"{RECON_FILE}: concurrencyPolicy must be Forbid — two concurrent reconcilers "
|
||||
f"would double the write attempts against the IdP during an outage recovery")
|
||||
check(re.fullmatch(r"\S+ \S+ \S+ \S+ \S+", cj["spec"].get("schedule", "")),
|
||||
f"{RECON_FILE}: schedule {cj['spec'].get('schedule')!r} is not a 5-field cron")
|
||||
|
||||
# ===== (6) THE BOUND FLOW IS STRUCTURALLY UNWRITABLE — asserted against the SOURCE
|
||||
#
|
||||
# This component converges the browser flow BLUE/GREEN: it builds the target structure
|
||||
# under a NEW, unused alias, verifies it, then flips the realm's browserFlow in ONE
|
||||
# atomic PUT. It therefore NEEDS flow-create and execution-add endpoints, so unlike the
|
||||
# first design of this check the mere presence of a flow write is no longer the thing to
|
||||
# forbid. What must hold instead is narrower and stronger:
|
||||
#
|
||||
# (a) nothing may be DELETED, ever — the superseded flow is the rollback target;
|
||||
# (b) the single-execution and config-edit endpoints stay off the list — requirements
|
||||
# are set through the flow-scoped endpoint, which the guard in (d) screens;
|
||||
# (c) no required-action write may reach any alias but the one named LITERALLY;
|
||||
# (d) and the guard the allow-list CANNOT express must exist in the source: a
|
||||
# method-aware screen that refuses any flow write targeting the CURRENTLY BOUND
|
||||
# flow's subtree, FAIL-CLOSED until that binding has been read.
|
||||
#
|
||||
# (d) is the load-bearing one. Without it the reconciler could mutate the flow users are
|
||||
# logging in through, and a run dying mid-restructure would leave a HALF-BUILT FLOW BOUND
|
||||
# AS THE LOGIN PATH for Grafana, Gitea, Nextcloud, Headlamp, Semaphore, PiHole, Crafty,
|
||||
# the *arr stack AND kubectl. This repo has already had a total SSO outage from a realm
|
||||
# change (2026-09-06).
|
||||
allow = tuple_literal(script, "_ALLOWED") or ()
|
||||
check(bool(allow),
|
||||
f"{RECON_FILE}: could not parse the _ALLOWED endpoint allow-list out of the "
|
||||
f"script. This check's central assertions about the write surface cannot be made, "
|
||||
f"so it must FAIL rather than pass silently")
|
||||
if allow:
|
||||
check(all(isinstance(e, tuple) and len(e) == 2 for e in allow),
|
||||
f"{RECON_FILE}: _ALLOWED is not a tuple of (method, path) pairs. A path-only "
|
||||
f"allow-list cannot distinguish reading a flow from writing one, which this "
|
||||
f"component's whole safety argument rests on")
|
||||
writes = [(m, p) for m, p in allow if m in ("PUT", "POST", "PATCH", "DELETE")]
|
||||
# (a) nothing is ever deleted.
|
||||
check(not [(m, p) for m, p in allow if m == "DELETE"],
|
||||
f"{RECON_FILE}: the allow-list contains a DELETE method. Nothing this program "
|
||||
f"does requires deleting anything: a superseded flow is left unbound as the "
|
||||
f"ROLLBACK TARGET, and DELETE /admin/realms/<realm> deletes the realm")
|
||||
# (b) the endpoints that would let it edit live objects in place.
|
||||
banned = [(m, p) for m, p in writes
|
||||
if re.fullmatch(r"/admin/realms/\[\^/\]\+/authentication/executions/"
|
||||
r"\[\^/\]\+", p)
|
||||
or re.fullmatch(r"/admin/realms/\[\^/\]\+/authentication/config/\[\^/\]\+",
|
||||
p)
|
||||
or p.endswith("/authentication/register-required-action")]
|
||||
check(not banned,
|
||||
f"{RECON_FILE}: the allow-list permits {banned}. The single-execution update "
|
||||
f"and config-edit endpoints must stay off it: requirements are set through the "
|
||||
f"FLOW-SCOPED endpoint, which carries the flow alias in its path and is "
|
||||
f"therefore screenable against the bound subtree. An execution- or "
|
||||
f"config-id-scoped write is not, so it would be an unscreened hole")
|
||||
# (c) no required-action write may reach any other alias.
|
||||
ra_writes = [(m, p) for m, p in writes if "required-actions" in p]
|
||||
check(all(p.endswith("required-actions/" + ENROL_ACTION) for _, p in ra_writes),
|
||||
f"{RECON_FILE}: the allow-list permits a required-action write to an alias "
|
||||
f"other than {ENROL_ACTION!r}: {ra_writes}. Disabling CONFIGURE_TOTP is a "
|
||||
f"permanent lock-out of every platform-admin, because auth-otp-form in the "
|
||||
f"browser flow still demands a TOTP that could no longer be enrolled")
|
||||
check(not [p for _, p in ra_writes if "[^/]+" in p.split("required-actions/")[-1]],
|
||||
f"{RECON_FILE}: a required-action write path uses a WILDCARD alias. The alias "
|
||||
f"must be named LITERALLY so no other action is writable by construction")
|
||||
# Nothing outside the realm + authentication surface at all.
|
||||
stray = [(m, p) for m, p in allow
|
||||
if "/users" in p or "/groups" in p or "/roles" in p or "/clients" in p]
|
||||
check(not stray,
|
||||
f"{RECON_FILE}: the allow-list reaches {stray}. Users, groups and roles must "
|
||||
f"be unreachable (prod's real user and the media-users / platform-admins / "
|
||||
f"viewers groups must survive every run); clients are the SIBLING "
|
||||
f"reconciler's surface and the two must not overlap")
|
||||
|
||||
# ---- (d) THE PROTECTED-SUBTREE GUARD EXISTS, IS WIRED, AND IS FAIL-CLOSED ----------
|
||||
# Asserted against the SOURCE, because this is the property the blue/green design rests
|
||||
# on and a comment claiming it is worth nothing. Three things must all be true.
|
||||
check(re.search(r"^def _screen_write\(", script, re.M) is not None,
|
||||
f"{RECON_FILE}: the script has no _screen_write() function. That is the guard "
|
||||
f"that refuses any flow write targeting the CURRENTLY BOUND flow's subtree — the "
|
||||
f"one thing the allow-list cannot express, because its write entries are "
|
||||
f"alias-parameterised. Without it this component can mutate the live login path")
|
||||
# It must be CALLED from the single choke point, not merely defined.
|
||||
check(re.search(r"_screen_write\(method, path", script) is not None,
|
||||
f"{RECON_FILE}: _screen_write() is defined but never called from _http() with the "
|
||||
f"request's method and path. A guard that is not on the request path is decoration")
|
||||
# FAIL-CLOSED: the protected sets must start undetermined, and the reset that runs at
|
||||
# the top of every run must put them back there.
|
||||
reset_fn = re.search(r"def _reset_protected\(\):(.*?)(?=\n\S)", script, re.S)
|
||||
check(reset_fn is not None
|
||||
and "_PROTECTED_FLOW_ALIASES = None" in reset_fn.group(1)
|
||||
and "_PROTECTED_EXECUTION_IDS = None" in reset_fn.group(1),
|
||||
f"{RECON_FILE}: _reset_protected() does not set both protected sets to None. They "
|
||||
f"must start each run UNDETERMINED, because while they are, every flow write is "
|
||||
f"refused — that is the fail-closed half of the guard, and it is what stops a "
|
||||
f"build running before the live binding has been read")
|
||||
check(re.search(r"_reset_protected\(\)", script.split("def _selftest")[0]) is not None,
|
||||
f"{RECON_FILE}: _reset_protected() is never called outside the selftest, so the "
|
||||
f"protected sets are not reset per run and a previous run's binding could be "
|
||||
f"trusted for this one")
|
||||
# The create-flow endpoint carries NO alias in its path, so it must be screened on its
|
||||
# BODY or it escapes the guard entirely. (Found by the selftest; recorded here so a
|
||||
# future edit cannot silently undo it.)
|
||||
check("_FLOW_CREATE_PATH" in script,
|
||||
f"{RECON_FILE}: there is no separate screen for POST .../authentication/flows. "
|
||||
f"That endpoint carries no alias in its path — the alias is in the request BODY — "
|
||||
f"so without a body-based screen it bypasses the protected-subtree guard "
|
||||
f"altogether")
|
||||
# And the one function allowed to change the live binding must be distinct from the
|
||||
# policy write, which must refuse to.
|
||||
# FINDING 2026-10-29: sub-flows are named by displayName in 26.0.6 (recurseExecutions
|
||||
# never sets `alias` on a sub-flow row). Reading `alias` shrank the protected subtree to
|
||||
# the top-level flow alone while every selftest passed against a stub that returned it.
|
||||
check(re.search(r'^def subflow_alias\(ex\):\n(?:.*\n)*? name = ex\.get\("displayName"\)',
|
||||
script, re.M) is not None,
|
||||
f"{RECON_FILE}: subflow_alias() no longer reads displayName. Keycloak 26.0.6's "
|
||||
f"executions listing names a sub-flow ONLY by displayName; reading `alias` yields "
|
||||
f"None and the protected subtree silently shrinks to the top-level flow")
|
||||
# FINDING 2026-10-29: the requirement PUT's real target is body["id"].
|
||||
check("_EXEC_UPDATE_PATH" in script and "_BUILD_EXECUTION_IDS" in script,
|
||||
f"{RECON_FILE}: the requirement PUT (PUT /flows/<alias>/executions) is no longer "
|
||||
f"screened on its body id. updateExecutions resolves the execution realm-wide by id "
|
||||
f"and ignores the path alias, so a path-only screen lets it write the bound flow")
|
||||
check("def declared_bind(" in script and "BIND_FILE" in script,
|
||||
f"{RECON_FILE}: the reconciler no longer reads git's bind-browser-flow switch")
|
||||
check("def bind_browser_flow(" in script,
|
||||
f"{RECON_FILE}: there is no bind_browser_flow() function. The atomic flip must be "
|
||||
f"one named, auditable operation, not an incidental field in some other write")
|
||||
check(re.search(r"never rebinds the browser flow", script) is not None,
|
||||
f"{RECON_FILE}: the policy write no longer refuses to change browserFlow. "
|
||||
f"PUT /admin/realms/<realm> DOES read rep.getBrowserFlow(), so the policy pass "
|
||||
f"can rebind the login path by accident — to an UNVERIFIED flow")
|
||||
|
||||
# ======================================== (7) the SIBLING reconciler is NOT widened
|
||||
if (ROOT / SIBLING_FILE).is_file():
|
||||
sib_docs = [d for d in load(SIBLING_FILE) if d]
|
||||
sib_cm = next((d for d in sib_docs if d.get("kind") == "ConfigMap"), None)
|
||||
sib_script = next(iter((sib_cm.get("data") or {}).values()), "") if sib_cm else ""
|
||||
sib_allow = tuple_literal(sib_script, "_ALLOWED") or ()
|
||||
check(bool(sib_allow),
|
||||
f"{SIBLING_FILE}: could not parse its _ALLOWED list, so this check cannot "
|
||||
f"prove the sibling was not widened")
|
||||
if sib_allow:
|
||||
# The sibling's list is PATH-ONLY by design. Its selftest asserts it refuses the
|
||||
# whole-realm endpoint and the /authentication tree; assert the same here so a
|
||||
# future edit that "just adds the realm endpoint there instead" is caught.
|
||||
bad = [p for p in sib_allow
|
||||
if isinstance(p, str)
|
||||
and (re.fullmatch(r"/admin/realms/\[\^/\]\+", p)
|
||||
or "authentication" in p or "/users" in p or "/groups" in p
|
||||
or "/roles" in p)]
|
||||
check(not bad,
|
||||
f"{SIBLING_FILE}: its allow-list was WIDENED to {bad}. That reconciler's "
|
||||
f"stated guarantee — prod's real user and the media-users / "
|
||||
f"platform-admins / viewers groups survive every run — is expressed as "
|
||||
f"endpoints it CANNOT reach, and its own --selftest asserts the refusal of "
|
||||
f"/users, /groups, /roles, /authentication and the whole-realm endpoint. "
|
||||
f"The passkey work is a SIBLING program with its own narrow list precisely "
|
||||
f"so that guarantee stays literally true. Revert the widening")
|
||||
|
||||
# ============================== (8) the reconciler's BEHAVIOUR, proven by its selftest
|
||||
if script:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "reconcile.py"
|
||||
p.write_text(script)
|
||||
proc = subprocess.run([sys.executable, str(p), "--selftest"],
|
||||
capture_output=True, text=True, timeout=300)
|
||||
tail = (proc.stdout + proc.stderr).strip().splitlines()
|
||||
check(proc.returncode == 0,
|
||||
"the passkey reconciler's --selftest FAILED (compare-then-write / "
|
||||
"idempotence / strict read-modify-write / the RpEntityName gate / "
|
||||
"flow-is-unwritable / the conditional-OTP regression / fail-closed):\n"
|
||||
" " + "\n ".join(tail[-14:]))
|
||||
if proc.returncode == 0:
|
||||
print(" passkey reconciler --selftest: " + (tail[-1] if tail else "(none)"))
|
||||
|
||||
# ---- (8b) THE SELFTEST IS FALSIFIABLE, not decorative ----------------------
|
||||
# A passing suite proves nothing unless it can be shown to FAIL. mutations.py
|
||||
# injects plausible regressions one at a time -- a partial policy PUT, a
|
||||
# rebind of browserFlow, a widened allow-list that makes the flow writable, a
|
||||
# fail-open on an unreadable input -- and requires --selftest to catch every
|
||||
# one. This matters more than usual here: the whole incident class is a
|
||||
# component that REPORTED SUCCESS while the realm was broken, and a green test
|
||||
# that cannot fail is the same lie one level up.
|
||||
mut = DIR_PATH / "mutations.py"
|
||||
if mut.is_file():
|
||||
mproc = subprocess.run([sys.executable, str(mut), str(p)],
|
||||
capture_output=True, text=True, timeout=900)
|
||||
mtail = (mproc.stdout + mproc.stderr).strip().splitlines()
|
||||
check(mproc.returncode == 0,
|
||||
"the passkey reconciler's --selftest is NOT FALSIFIABLE — at least "
|
||||
"one injected regression survived it, so the suite does not prove "
|
||||
"what it claims:\n " + "\n ".join(mtail[-16:]))
|
||||
if mproc.returncode == 0:
|
||||
print(" passkey reconciler mutations: "
|
||||
+ (mtail[-1] if mtail else "(no output)"))
|
||||
else:
|
||||
check(False,
|
||||
"conformance/tests/l0-static/keycloak-passkey-reconciler/mutations.py "
|
||||
"is missing — nothing proves the --selftest can fail, and an "
|
||||
"unfalsifiable proof is the failure mode this component exists to end")
|
||||
|
||||
# ======================================================= (9) this check is load-bearing
|
||||
L0 = (ROOT / "ci" / "l0.sh").read_text() if (ROOT / "ci" / "l0.sh").is_file() else ""
|
||||
GATE = (ROOT / "ci" / "gate.sh").read_text() if (ROOT / "ci" / "gate.sh").is_file() else ""
|
||||
TOUCHED = ((ROOT / "ci" / "touched-requirements.sh").read_text()
|
||||
if (ROOT / "ci" / "touched-requirements.sh").is_file() else "")
|
||||
SELF_YAML = DIR_PATH / "chainsaw-test.yaml"
|
||||
SELF = SELF_YAML.read_text() if SELF_YAML.is_file() else ""
|
||||
check(re.search(r'^\s*conformance\.home\.k3s/scope:\s*["\']?whole-repo', SELF, re.M)
|
||||
is not None,
|
||||
"keycloak-passkey-reconciler/chainsaw-test.yaml does not declare "
|
||||
'conformance.home.k3s/scope: "whole-repo", so the merge gate selects this check '
|
||||
"only when a change happens to attribute one of the requirements it is labelled "
|
||||
"with. This invariant spans platform/, platform-config/, ci/ and clusters/, so a "
|
||||
"change that breaks it need not attribute R-IDP-03/04 — which is how 12 unusable "
|
||||
"prod OIDC clients sat behind an entirely green cluster on 2026-09-06")
|
||||
check("--include-scope" in L0,
|
||||
"ci/l0.sh no longer passes --include-scope to the conformance runner, so the L0 "
|
||||
"stage is a purely requirement-scoped run again and the scope annotation is inert "
|
||||
"decoration — this check would then run only by coincidence")
|
||||
check(re.search(r"^for s in .*\bl0\b.*; do", GATE, re.M)
|
||||
and re.search(r"fail\).*BLOCK=1", GATE),
|
||||
"ci/gate.sh does not set the merge block when the l0 stage fails")
|
||||
check(re.search(r"^\s*platform/keycloak/\*\)", TOUCHED, re.M),
|
||||
"ci/touched-requirements.sh has no platform/keycloak/* arm, so editing the realm "
|
||||
"import or the reconciler would attribute NO requirement and the keycloak tests "
|
||||
"would be filtered out of the merge gate's touched subset")
|
||||
|
||||
# =============================================================================
|
||||
if failures:
|
||||
print("CHECK FAIL: %d assertion(s) failed" % len(failures), file=sys.stderr)
|
||||
for f in failures:
|
||||
print(" FAIL %s" % f, file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print("keycloak-passkey-reconciler: all assertions passed "
|
||||
"(desired state derived from the realm import, one substitution pass, write "
|
||||
"surfaces agree, the BOUND flow is unwritable and fail-closed, nothing is "
|
||||
"deletable, sibling allow-list unwidened, selftest green and falsifiable)")
|
||||
PY
|
||||
@@ -0,0 +1,516 @@
|
||||
#!/usr/bin/env python3
|
||||
# =============================================================================
|
||||
# mutations.py -- prove the passkey reconciler's --selftest IS FALSIFIABLE.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# ---------------
|
||||
# A selftest that passes proves nothing on its own: a selftest asserting `True ==
|
||||
# True` also passes. This reconciler exists because a one-shot realm import
|
||||
# REPORTED SUCCESS while the realm was never converged (Done=True / Started=False
|
||||
# with the whole passkey block inert, measured on prod 2026-10-18), and the same
|
||||
# trap applies one level up -- a green test suite that cannot fail is the same lie
|
||||
# in a different place.
|
||||
#
|
||||
# It matters more here than for the sibling client reconciler, because this program
|
||||
# CONSTRUCTS AUTHENTICATION FLOWS and then REBINDS THE REALM'S LOGIN PATH. An
|
||||
# untested safety property on that surface is not a documentation gap, it is an
|
||||
# outage waiting for a schedule tick.
|
||||
#
|
||||
# So this injects a series of PLAUSIBLE regressions into the shipped script, one at
|
||||
# a time, and requires that --selftest FAILS for every one. A mutation that
|
||||
# SURVIVES means the suite does not actually check that property, and the check
|
||||
# fails, naming it.
|
||||
#
|
||||
# THE FIRST SIX ARE THE BLUE/GREEN SAFETY ARGUMENT ITSELF. The design builds a new
|
||||
# flow under an unused alias, verifies it, and flips `browserFlow` in one atomic
|
||||
# write -- never mutating the flow users are logging in through. Each of those
|
||||
# properties is enforced in code; these mutations are the proof that the enforcement
|
||||
# is load-bearing rather than commentary. Several model the obvious "simplification"
|
||||
# someone will propose:
|
||||
# * "just build it under the declared alias, the suffix is ugly"
|
||||
# * "bind it first, then check -- the check is just belt and braces"
|
||||
# * "the protected-alias screen is redundant, the allow-list covers it"
|
||||
# Every one of those would be a total SSO outage waiting to happen.
|
||||
#
|
||||
# ANCHORS ARE AUTHORED AGAINST THE EXTRACTED SCRIPT, not against the YAML manifest.
|
||||
# The sibling mutations.py dedents by four because its anchors are copied out of the
|
||||
# ConfigMap block scalar; these are copied out of the script as `python3 -` sees it,
|
||||
# which is one less transformation to get wrong. check.sh passes the extracted file,
|
||||
# so what is proven falsifiable is byte-for-byte what runs on the cluster either way.
|
||||
#
|
||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||
#
|
||||
# Runtime is ~85s serial (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||
# verdicts are independent of the pool size.
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
# (name, the mistake being modelled, anchor, replacement)
|
||||
MUTATIONS: list[tuple[str, str, str, str]] = [
|
||||
# ---------------------------------------------------------------- blue/green safety
|
||||
("build INTO the currently bound flow",
|
||||
"'the suffix is ugly, just build under the declared alias' -- every "
|
||||
"create-execution and set-requirement call would then land on the flow users are "
|
||||
"logging in through, and a run dying part-way leaves a HALF-BUILT LOGIN PATH for "
|
||||
"every SSO app and kubectl. This is the single regression the whole design exists "
|
||||
"to make impossible",
|
||||
" def mapped(alias):\n"
|
||||
" return \"%s-%s\" % (alias, suffix)",
|
||||
" def mapped(alias):\n return alias"),
|
||||
|
||||
("the protected-subtree screen is never called",
|
||||
"'the allow-list already covers it' -- it does not: the allow-list's write entries "
|
||||
"are alias-parameterised, so only this screen can say 'but not the flow that is "
|
||||
"bound'. Without it the reconciler may write anywhere in the authentication tree",
|
||||
" _screen_write(method, path, body if isinstance(body, dict) else None)",
|
||||
" pass"),
|
||||
|
||||
("the per-run reset makes the protected sets EMPTY, not UNDETERMINED",
|
||||
"reset them to frozenset() so the code reads more simply -- it silently removes "
|
||||
"the fail-closed property, so a build that ran before the live binding had been "
|
||||
"read would be PERMITTED, and could write straight into the live flow. (The "
|
||||
"module-level initial value is deliberately NOT the anchor here: run() resets "
|
||||
"these at the start of every run, so the initialiser is unreachable and mutating "
|
||||
"it would prove nothing.)",
|
||||
" global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n"
|
||||
" _PROTECTED_FLOW_ALIASES = None\n"
|
||||
" _PROTECTED_EXECUTION_IDS = None",
|
||||
" global _PROTECTED_FLOW_ALIASES, _PROTECTED_EXECUTION_IDS, _BUILD_EXECUTION_IDS\n"
|
||||
" _PROTECTED_FLOW_ALIASES = frozenset()\n"
|
||||
" _PROTECTED_EXECUTION_IDS = frozenset()"),
|
||||
|
||||
("bind BEFORE verifying the build",
|
||||
"'bind it first, the readback is just belt and braces' -- it is not: the readback "
|
||||
"is what catches a build that was accepted but came out wrong, including the "
|
||||
"conditional-OTP config landing under the wrong key. Binding first makes that the "
|
||||
"live login path before anyone notices",
|
||||
" verify = compare_shape(final_shape, expected, base, realm, token, configs,\n"
|
||||
" \"the newly built flow %r\" % target)",
|
||||
" verify = []"),
|
||||
|
||||
("a failed build is reported as SUCCESS",
|
||||
"swallow the build error and return a converged result -- the realm is then left "
|
||||
"on the old flow (which blue/green makes safe) but the run says it converged, so "
|
||||
"nobody ever learns the passkey flow was never built. That is the one-shot "
|
||||
"import's fail-open, reintroduced. NOTE: simply removing the early return here "
|
||||
"is NOT a regression -- verify-before-bind catches a half-built flow anyway, which "
|
||||
"is defence in depth working as intended -- so the mutation targets the reporting",
|
||||
" problems.append(\n"
|
||||
" \"flow: building %r FAILED (%s). Nothing was bound, so login is \"\n"
|
||||
" \"UNAFFECTED -- the old flow is still live. The partial flow is left in \"\n"
|
||||
" \"place, unbound and harmless\" % (candidate, e))\n"
|
||||
" return 0, problems",
|
||||
" return 1, []"),
|
||||
|
||||
("an existing candidate flow is reused WITHOUT verifying it",
|
||||
"'if the alias exists we already built it' -- a half-built leftover from a crashed "
|
||||
"run lives under exactly that alias, so this binds a partially constructed login "
|
||||
"path",
|
||||
" if cand_shape is not None and not compare_shape(\n"
|
||||
" cand_shape, expected, base, realm, token, configs,\n"
|
||||
" \"the existing flow %r\" % candidate):",
|
||||
" if cand_shape is not None:"),
|
||||
|
||||
# ------------------------------------- 2026-10-29 review findings, each made falsifiable
|
||||
("sub-flows identified by `alias` again",
|
||||
"read `alias` off a sub-flow row, as the first version did -- Keycloak 26.0.6's "
|
||||
"recurseExecutions sets only displayName (and flowId) on a sub-flow row, so every "
|
||||
"sub-flow resolves to nothing and the protected subtree is the top-level flow alone: "
|
||||
"a write into browser-mfa-forms would NOT be refused",
|
||||
" name = ex.get(\"displayName\")\n"
|
||||
" return name if isinstance(name, str) and name else None",
|
||||
" name = ex.get(\"alias\")\n"
|
||||
" return name if isinstance(name, str) and name else None"),
|
||||
|
||||
("an unnameable sub-flow is tolerated",
|
||||
"carry on when a sub-flow row has no displayName -- the protected subtree is then "
|
||||
"SMALLER than the live tree, which is the fail-open the guard exists to prevent",
|
||||
" if name is None:\n"
|
||||
" return None, None",
|
||||
" if name is None:\n"
|
||||
" name = \"\""),
|
||||
|
||||
("the requirement PUT is screened on its path only",
|
||||
"trust the alias in PUT /flows/<alias>/executions -- updateExecutions resolves "
|
||||
"body.id realm-wide and never checks it belongs to <alias>, so a candidate alias in "
|
||||
"the path can carry a BOUND execution's id in the body",
|
||||
" targets.append((\"execution\", (body or {}).get(\"id\")))",
|
||||
" pass"),
|
||||
|
||||
("execution-addressed writes are not held to the build tree",
|
||||
"'outside the bound subtree is enough' -- it is not: every OTHER flow in the realm is "
|
||||
"outside it too, so a stray id would edit an unrelated flow",
|
||||
" if what == \"execution\" and (_BUILD_EXECUTION_IDS is None\n"
|
||||
" or target not in _BUILD_EXECUTION_IDS):",
|
||||
" if False:"),
|
||||
|
||||
("the requirement PUT may carry any key",
|
||||
"let the body carry flowId / displayName -- the same handler then RENAMES whatever "
|
||||
"flow flowId names, bound or not",
|
||||
" if extra:\n"
|
||||
" raise Refused(",
|
||||
" if False:\n"
|
||||
" raise Refused("),
|
||||
|
||||
("git's bind switch is ignored",
|
||||
"bind whenever verification passes, as the first version did -- the first live use of "
|
||||
"an untested login flow becomes a schedule tick, and an operator's rollback is "
|
||||
"undone within 15 minutes, forever",
|
||||
" bind = git_bind and not no_bind",
|
||||
" bind = not no_bind"),
|
||||
|
||||
("bind=false still binds inside the flow pass",
|
||||
"keep the flag but forget to honour it where the bind happens",
|
||||
" if not bind:\n"
|
||||
" # THE BIND SWITCH IS OFF.",
|
||||
" if False:\n"
|
||||
" # THE BIND SWITCH IS OFF."),
|
||||
|
||||
("an absent bind switch means bind",
|
||||
"default to true when the key is missing -- a ConfigMap predating the switch, or a "
|
||||
"deleted line, would silently put an untested flow live",
|
||||
" except FileNotFoundError:\n"
|
||||
" return False",
|
||||
" except FileNotFoundError:\n"
|
||||
" return True"),
|
||||
|
||||
("an invalid bind switch is guessed at",
|
||||
"read anything that is not \"true\" as false -- a typo then passes silently instead of "
|
||||
"stopping the run, and the operator believes the flow is live",
|
||||
" if raw == \"false\":\n"
|
||||
" return False\n"
|
||||
" raise ValueError(",
|
||||
" return False\n"
|
||||
" raise ValueError("),
|
||||
|
||||
("bind=false rebuilds beside a hand-edited bound flow",
|
||||
"treat the hand-edited bound candidate as a 'half-built leftover' -- with the switch "
|
||||
"off this builds an -r2 next to the operator's hotfix on every tick",
|
||||
" if candidate in have and candidate == bound and not bind:",
|
||||
" if False:"),
|
||||
|
||||
# ------------------------------------------------------------------- idempotence
|
||||
("sub-flow aliases are compared as names",
|
||||
"compare the sub-flow alias like any other field -- the flows this program builds "
|
||||
"carry a hash suffix on every alias, so a built flow could NEVER be recognised as "
|
||||
"converged: it would be rebuilt and the login path reflipped every fifteen "
|
||||
"minutes, forever",
|
||||
" out.append((level, ex[\"kind\"], \"\" if is_flow else ex[\"name\"],\n"
|
||||
" ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))",
|
||||
" out.append((level, ex[\"kind\"], ex[\"name\"],\n"
|
||||
" ex[\"requirement\"], ex.get(\"configAlias\") or \"\", ex[\"name\"]))"),
|
||||
|
||||
("the converged short-circuit is removed",
|
||||
"drop the 'bound flow already matches git' early return -- every run rebuilds and "
|
||||
"reflips, which is a permanent write loop against the IdP and churns the realm's "
|
||||
"login path on a schedule",
|
||||
" if not drift:\n"
|
||||
" log(\"flow ok (the bound flow %r matches git: %d execution(s), no \"\n"
|
||||
" \"write)\" % (bound, len(expected)))\n"
|
||||
" return 0, problems",
|
||||
" if False:\n return 0, problems"),
|
||||
|
||||
("the execution priority is dropped from the requirement write",
|
||||
"omit `priority` from the body -- Keycloak's updateExecutions deserialises a "
|
||||
"missing int as 0 and compares it against the stored priority, so this RESHUFFLES "
|
||||
"the execution order while only a requirement change was intended. The passkey "
|
||||
"branch would end up in the wrong place in the tree",
|
||||
" body={\"id\": ex.get(\"id\"), \"requirement\": want_req,\n"
|
||||
" \"priority\": ex.get(\"priority\") or 0})",
|
||||
" body={\"id\": ex.get(\"id\"), \"requirement\": want_req})"),
|
||||
|
||||
("the alias budget is unbounded",
|
||||
"loop forever looking for a free alias -- a persistently failing build then creates "
|
||||
"unbounded flow sprawl in the realm, and nothing here can delete any of it",
|
||||
" for attempt in range(1, MAX_BUILD_ATTEMPTS + 1):",
|
||||
" for attempt in range(1, 10000):"),
|
||||
|
||||
("fail-open when the bound flow cannot be read",
|
||||
"carry on when the live flow's executions are unreadable -- the program then cannot "
|
||||
"know which flows are live, so it cannot prove a build would miss them, and it "
|
||||
"would build with the protected sets still undetermined",
|
||||
" return 0, [\n"
|
||||
" \"flow: the realm binds browserFlow to %r but its executions could not \"\n"
|
||||
" \"be read, so this program cannot establish which flows are live. Flow \"\n"
|
||||
" \"writes stay FAIL-CLOSED for this run -- nothing was built or bound\"\n"
|
||||
" % bound]",
|
||||
" _set_protected(set(), set())\n bound_shape, bound_raw = None, None"),
|
||||
|
||||
# ------------------------------------------------------- the conditional-OTP regression
|
||||
("authenticator config contents no longer compared",
|
||||
"check only that a config is BOUND, not what it says -- this is THE conditional-OTP "
|
||||
"regression: the original bug wrote the role under "
|
||||
"`condition.evaluated.user.role` instead of `condUserRole`, so the role resolved "
|
||||
"to null, the REQUIRED conditional step threw 'Invalid role name submitted: null' "
|
||||
"and every platform-admin got a 403 from every SSO-gated app",
|
||||
" if str(have_cfg.get(key)) != str(want_cfg[key]):",
|
||||
" if False:"),
|
||||
|
||||
("undeclared config keys are ignored",
|
||||
"stop reporting keys the realm holds and git does not -- the stale "
|
||||
"`condition.evaluated.user.role` key is the signature of the original null-role "
|
||||
"bug, and a flow carrying both keys would look correct",
|
||||
" extra = sorted(k for k in have_cfg if k not in want_cfg)",
|
||||
" extra = []"),
|
||||
|
||||
("flow structure divergence detected but not reported",
|
||||
"leave the per-step comparison out -- the realm's login flow could be the old "
|
||||
"pre-passkey shape forever and every run would report the realm converged, which "
|
||||
"is the exact fail-open of the one-shot import this program replaces",
|
||||
" if observed[idx][:4] != expected[idx][:4]:",
|
||||
" if False:"),
|
||||
|
||||
("the browserFlow BINDING is no longer compared",
|
||||
"stop noticing that a correct flow is not the one bound -- a perfect, fully-built "
|
||||
"passkey flow that nothing logs in through changes nothing at all",
|
||||
" if bound_shape is not None:\n"
|
||||
" drift = compare_shape(bound_shape, expected, base, realm, token, configs,",
|
||||
" if False:\n"
|
||||
" drift = compare_shape(bound_shape, expected, base, realm, token, configs,"),
|
||||
|
||||
# ------------------------------------------------------------- realm / policy writes
|
||||
("partial policy PUT instead of read-modify-write",
|
||||
"send only the drifted policy keys -- Keycloak 26.0.6 rebuilds the WHOLE "
|
||||
"passwordless policy from the representation and substitutes its own DEFAULT for "
|
||||
"every key the body omits, so a partial PUT silently resets everything it did not "
|
||||
"mention while returning 204",
|
||||
" body = dict(realm_rep)\n"
|
||||
" for key in drift:\n"
|
||||
" body[key] = wanted[key]",
|
||||
" body = {RP_ENTITY_NAME: realm_rep.get(RP_ENTITY_NAME),\n"
|
||||
" BROWSER_FLOW: realm_rep.get(BROWSER_FLOW)}\n"
|
||||
" for key in drift:\n"
|
||||
" body[key] = wanted[key]"),
|
||||
|
||||
("the policy pass rebinds browserFlow as a side effect",
|
||||
"'while we are here, let the policy write fix the flow binding too' -- the realm "
|
||||
"PUT does read rep.getBrowserFlow(), so this is genuinely in reach, and it would "
|
||||
"rebind the login path to an UNVERIFIED flow from a pass whose stated surface is "
|
||||
"the passkey policy",
|
||||
" if body.get(BROWSER_FLOW) != realm_rep.get(BROWSER_FLOW):\n"
|
||||
" raise RuntimeError(",
|
||||
" body[BROWSER_FLOW] = \"browser\"\n"
|
||||
" if False:\n"
|
||||
" raise RuntimeError("),
|
||||
|
||||
("the RpEntityName gate removed from desired-state validation",
|
||||
"accept a desired state with an empty RpEntityName -- Keycloak 26.0.6 applies the "
|
||||
"passwordless policy ONLY when that field is non-empty, so every realm PUT would "
|
||||
"be accepted with 204 and change nothing: reported success, zero convergence",
|
||||
" if not policy.get(RP_ENTITY_NAME):\n"
|
||||
" raise ValueError(",
|
||||
" if False:\n raise ValueError("),
|
||||
|
||||
("the RpEntityName gate removed from the write path",
|
||||
"drop the second, defence-in-depth guard -- it is unreachable while the "
|
||||
"desired-state validation holds, which is exactly why deleting it would otherwise "
|
||||
"be an invisible regression",
|
||||
" if not body.get(RP_ENTITY_NAME):\n"
|
||||
" raise RuntimeError(",
|
||||
" if False:\n raise RuntimeError("),
|
||||
|
||||
("no readback verification on the policy write",
|
||||
"trust the 204 -- a server that accepts the PUT and does not apply it is reported "
|
||||
"as REPAIRED, which is 'reports success while the realm never converged' again, "
|
||||
"and it is precisely what the RpEntityName gate does when it is missed",
|
||||
" still = policy_drift(after, wanted)",
|
||||
" still = {}"),
|
||||
|
||||
("AAGUID comparison made order-sensitive",
|
||||
"compare the AAGUID allow-list in the order Keycloak returns it -- an allow-list "
|
||||
"is a set membership test, so an order-sensitive comparison rewrites a CONVERGED "
|
||||
"realm on every run forever while every run still reports success",
|
||||
" if stored != sorted(declared_value):\n"
|
||||
" drift[key] = (stored, sorted(declared_value))",
|
||||
" if stored != declared_value:\n"
|
||||
" drift[key] = (stored, declared_value)"),
|
||||
|
||||
("signature-algorithm order treated as insignificant",
|
||||
"sort the signature algorithms before comparing -- it is a PREFERENCE LIST handed "
|
||||
"to the authenticator, so ES256-then-RS256 is not the same declaration as "
|
||||
"RS256-then-ES256, and a reorder in the realm would never be corrected",
|
||||
" if stored != list(declared_value):",
|
||||
" if sorted(stored) != sorted(declared_value):"),
|
||||
|
||||
("unmanaged policy keys forced to a value",
|
||||
"iterate every writable key instead of only the declared ones -- a "
|
||||
"webAuthnPolicyPasswordless field the realm holds and git does not declare would "
|
||||
"be overwritten, turning 'not declared' into 'must equal my default'",
|
||||
" for key in sorted(wanted):\n declared_value = wanted[key]",
|
||||
" for key in POLICY_FIELDS:\n declared_value = wanted.get(key, \"\")"),
|
||||
|
||||
# ------------------------------------------------------------------ required actions
|
||||
("required-action PUT sends only the two drifted fields",
|
||||
"a minimal body looks safer -- but updateRequiredAction REPLACES the provider from "
|
||||
"the representation, so name, priority and config are all blanked: the enrolment "
|
||||
"action keeps working but loses its priority and display name",
|
||||
" body = dict(rep)\n for key in drift:\n body[key] = wanted[key]",
|
||||
" body = {}\n for key in drift:\n body[key] = wanted[key]"),
|
||||
|
||||
("the guarded default required actions are no longer audited",
|
||||
"stop checking that CONFIGURE_TOTP / UPDATE_PASSWORD / delete_credential still "
|
||||
"exist -- their disappearance is the signature of a requiredActions block in the "
|
||||
"realm import, which 26.0.6 treats as REPLACE, and it is a silent permanent "
|
||||
"lock-out of every platform-admin",
|
||||
" missing = [a for a in GUARDED_ACTIONS if a not in aliases]",
|
||||
" missing = []"),
|
||||
|
||||
# ----------------------------------------------------------------- inputs / coupling
|
||||
("the three passes are coupled",
|
||||
"abort the run at the first failing pass -- a policy write that fails for an "
|
||||
"unrelated reason would then suppress the browser-flow pass, which is the most "
|
||||
"safety-critical thing here",
|
||||
" problems.extend(found)\n if found:\n rc = 1",
|
||||
" problems.extend(found)\n if found:\n rc = 1\n"
|
||||
" break"),
|
||||
|
||||
("policy-key validation removed from the desired state",
|
||||
"accept any realm key from the ConfigMap -- an arbitrary realm attribute becomes "
|
||||
"writable through a file edit, so a hand-edited ConfigMap could set sslRequired, "
|
||||
"bruteForceProtected or any other top-level realm field on the live IdP",
|
||||
" if key not in POLICY_FIELDS:",
|
||||
" if False:"),
|
||||
|
||||
("fail-open on an unreadable desired state",
|
||||
"treat a missing ConfigMap as 'nothing declared' -- the run reports success while "
|
||||
"reconciling nothing, which is precisely the fail-open of the realm import this "
|
||||
"whole component exists to replace",
|
||||
" if not isinstance(raw, dict) or not raw:\n"
|
||||
" raise ValueError(\"%s does not hold a non-empty JSON object\" % path)",
|
||||
" if not isinstance(raw, dict) or not raw:\n return {}"),
|
||||
|
||||
("an undeclared authenticatorConfig reference is tolerated",
|
||||
"stop validating that every configAlias a flow references is actually declared -- "
|
||||
"the build then fails at the config-binding step, AFTER flows and executions have "
|
||||
"already been created, leaving leftovers on every run",
|
||||
" if ca and ca not in configs:\n"
|
||||
" raise ValueError(",
|
||||
" if False:\n raise ValueError("),
|
||||
|
||||
("the flow-cycle guard is removed",
|
||||
"drop the `seen` check -- a cyclic flow declaration then recurses until the "
|
||||
"interpreter dies, in a CronJob that retries",
|
||||
" if alias in seen:\n"
|
||||
" raise ValueError(\"flow %r is cyclic in the desired state\" % alias)",
|
||||
" if False:\n"
|
||||
" raise ValueError(\"flow %r is cyclic in the desired state\" % alias)"),
|
||||
]
|
||||
|
||||
|
||||
def mutation_jobs() -> int:
|
||||
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||
|
||||
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||
"""
|
||||
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||
if raw:
|
||||
try:
|
||||
n = int(raw)
|
||||
except ValueError:
|
||||
n = 0
|
||||
if n < 1:
|
||||
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||
return n
|
||||
try:
|
||||
cpus = len(os.sched_getaffinity(0))
|
||||
except (AttributeError, OSError):
|
||||
cpus = os.cpu_count() or 1
|
||||
return max(1, min(8, cpus))
|
||||
|
||||
|
||||
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(mutated_src)
|
||||
path = fh.name
|
||||
try:
|
||||
return subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
stdin=subprocess.DEVNULL)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||
return 2
|
||||
src = Path(sys.argv[1]).read_text()
|
||||
|
||||
try:
|
||||
jobs = mutation_jobs()
|
||||
except ValueError as e:
|
||||
print("mutations.py: %s" % e, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||
for name, why, old, new in MUTATIONS:
|
||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||
# moved and this mutation silently stopped proving anything.
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
planned.append((name, why,
|
||||
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
"code moved and this mutation no longer proves anything. "
|
||||
"Re-anchor it against the current source." % (name, n), None))
|
||||
continue
|
||||
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||
|
||||
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||
try:
|
||||
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||
for (_n, _w, _a, mutated) in planned]
|
||||
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||
if anchor_problem is not None:
|
||||
problems.append(anchor_problem)
|
||||
continue
|
||||
proc = fut.result()
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-54s %s" % (name, how))
|
||||
finally:
|
||||
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||
# serially; queued mutations are cancelled rather than run for nothing.
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
print()
|
||||
if problems:
|
||||
print("MUTATION CHECK FAIL: %d of %d mutation(s) not proven"
|
||||
% (len(problems), len(MUTATIONS)), file=sys.stderr)
|
||||
for p in problems:
|
||||
print(" FAIL %s" % p, file=sys.stderr)
|
||||
return 1
|
||||
print("MUTATION CHECK OK: all %d injected regressions were caught by --selftest "
|
||||
"(the suite is falsifiable, not decorative)" % killed)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — PiHole's DNS frontends admit the LAN and the VPN, and nothing public
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# loadBalancerSourceRanges is the only thing keeping PiHole from being an open resolver, and it
|
||||
# must also carry the WireGuard client subnet, because Cilium load-balances tunnel traffic on wg0
|
||||
# before the iptables MASQUERADE (so the source it tests is still 10.90.0.x). check.sh's header
|
||||
# states the assertions and what they do not prove.
|
||||
#
|
||||
# STATUS: EXECUTABLE. `check.sh` is a pure offline assertion script (standard library + PyYAML);
|
||||
# conformance/run.sh --layer l0 runs it directly and this Chainsaw Test wraps the SAME script.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: pihole-dns-source-ranges
|
||||
labels:
|
||||
conformance.home.k3s/R-DNS-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-DNS-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
# WHOLE-REPO SCOPE: ci/touched-requirements.sh maps apps/* to no R-DNS requirement, so a change
|
||||
# to these Services would otherwise select nothing.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
Every PiHole LoadBalancer Service lists both the LAN and the WireGuard client subnet in
|
||||
loadBalancerSourceRanges, and every range resolves to RFC1918, so VPN clients can resolve
|
||||
and PiHole stays unreachable from the internet.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-pihole-source-ranges
|
||||
description: >-
|
||||
Collect the PiHole LoadBalancer Services, resolve each source range through cluster-vars,
|
||||
and assert both required ranges are present and every range is private.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — pihole-dns-source-ranges
|
||||
# Serves: R-DNS-03 (LAN split-horizon DNS — spec docs/09).
|
||||
#
|
||||
# THE BUG CLASS THIS CLOSES. PiHole's LoadBalancer frontends are the only thing stopping it being
|
||||
# an open resolver, and their `loadBalancerSourceRanges` is pulled in two opposite directions:
|
||||
# * Too WIDE, and the resolver is open: a public range, or 0.0.0.0/0, in this list is the whole
|
||||
# restriction gone, with nothing else in the path to catch it (the CiliumNetworkPolicy admits
|
||||
# `world` on :53, because that is the only peer that can describe post-LoadBalancer traffic).
|
||||
# * Too NARROW, and VPN DNS dies: wg0 is a Cilium kube-proxy-replacement device, so a tunnel
|
||||
# client's query is load-balanced with its 10.90.0.x source intact, ahead of the iptables
|
||||
# MASQUERADE. A LAN-only list is predicted to drop it as DROP_NOT_IN_SRC_RANGE, and the
|
||||
# tunnel would still report a healthy handshake.
|
||||
#
|
||||
# GIVEN the repository as git tracks it,
|
||||
# WHEN every type=LoadBalancer Service in namespace `pihole` is collected and each source range
|
||||
# is resolved through cluster-vars (base value AND the prod overlay's replacement),
|
||||
# THEN (1) the frontends exist and cover both udp/53 and tcp/53;
|
||||
# (2) every frontend lists ${LAN_CIDR} and ${WG_CLIENT_CIDR};
|
||||
# (3) every range resolves to a valid IPv4 CIDR inside RFC1918 — so no 0.0.0.0/0, no public
|
||||
# range, and no unresolvable token that could hide one;
|
||||
# AND (4) the check is NON-VACUOUS: a real checkout and a floor on the assertion count.
|
||||
#
|
||||
# WHAT THIS DOES NOT PROVE: that a query over the tunnel is actually answered. That needs a
|
||||
# handshake from a real client; see platform/vpn-ingress/README.md.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
cd "${ROOT}"
|
||||
"${PYTHON}" - "${ROOT}" <<'PY'
|
||||
import ipaddress, re, subprocess, sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("check FAIL: PyYAML is required (same dependency as catalog/render.py)",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES: list[str] = []
|
||||
CHECKS = 0
|
||||
REQUIRED_TOKENS = ("LAN_CIDR", "WG_CLIENT_CIDR")
|
||||
RFC1918 = [ipaddress.ip_network(n) for n in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")]
|
||||
TOKEN = re.compile(r"^\$\{([A-Z0-9_]+)\}$")
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(msg)
|
||||
|
||||
|
||||
def docs_of(rel: str):
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all((ROOT / rel).read_text(errors="replace"))
|
||||
if isinstance(d, dict)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
FILES = [p for p in subprocess.run(["git", "-C", str(ROOT), "ls-files", "-z"],
|
||||
capture_output=True, text=True, check=True).stdout.split("\0")
|
||||
if p]
|
||||
YAML_FILES = [f for f in FILES if re.search(r"\.ya?ml$", f)]
|
||||
check(len(YAML_FILES) > 50, f"only {len(YAML_FILES)} YAML files seen — not a real checkout")
|
||||
|
||||
# ------------------------------------------------- every value a ${TOKEN} can take on this tree
|
||||
values: dict[str, set[str]] = {}
|
||||
for d in docs_of("clusters/base/cluster-vars.yaml"):
|
||||
if d.get("kind") == "ConfigMap":
|
||||
for k, v in (d.get("data") or {}).items():
|
||||
values.setdefault(k, set()).add(str(v))
|
||||
for d in docs_of("clusters/prod/kustomization.yaml"):
|
||||
for p in d.get("patches") or []:
|
||||
ops = p.get("patch") if isinstance(p, dict) else None
|
||||
ops = yaml.safe_load(ops) if isinstance(ops, str) else ops
|
||||
for op in ops if isinstance(ops, list) else []:
|
||||
m = re.fullmatch(r"/data/([A-Z0-9_]+)", str(op.get("path", "")))
|
||||
if m and op.get("op") in ("add", "replace"):
|
||||
values.setdefault(m.group(1), set()).add(str(op.get("value")))
|
||||
for tok in REQUIRED_TOKENS:
|
||||
check(values.get(tok), f"clusters/base/cluster-vars.yaml defines no {tok} (R-DNS-03)")
|
||||
|
||||
|
||||
def assert_private(where: str, cidr: str):
|
||||
try:
|
||||
net = ipaddress.ip_network(cidr, strict=False)
|
||||
except ValueError:
|
||||
check(False, f"{where}: {cidr!r} is not a valid CIDR (R-DNS-03)")
|
||||
return
|
||||
check(net.version == 4 and any(net.subnet_of(r) for r in RFC1918),
|
||||
f"{where}: {cidr} is not inside RFC1918, so it opens PiHole beyond the LAN and the VPN "
|
||||
"(R-DNS-03)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- the frontends
|
||||
frontends = []
|
||||
for rel in YAML_FILES:
|
||||
for d in docs_of(rel):
|
||||
md, spec = d.get("metadata") or {}, d.get("spec") or {}
|
||||
if d.get("kind") == "Service" and md.get("namespace") == "pihole" \
|
||||
and spec.get("type") == "LoadBalancer":
|
||||
frontends.append((rel, md.get("name"), spec))
|
||||
|
||||
protocols = {(p.get("protocol", "TCP"), p.get("port"))
|
||||
for _, _, spec in frontends for p in spec.get("ports") or []}
|
||||
for want in (("UDP", 53), ("TCP", 53)):
|
||||
check(want in protocols, f"no PiHole LoadBalancer Service serves {want[0]}/{want[1]} — this "
|
||||
"check's corpus has moved and must be re-derived (R-DNS-03)")
|
||||
|
||||
for rel, name, spec in frontends:
|
||||
where = f"{rel}: Service/{name}"
|
||||
ranges = spec.get("loadBalancerSourceRanges") or []
|
||||
check(ranges, f"{where} has no loadBalancerSourceRanges, so it answers anyone who can reach "
|
||||
"the VIP (R-DNS-03)")
|
||||
tokens = {m.group(1) for m in (TOKEN.match(str(r)) for r in ranges) if m}
|
||||
for tok in REQUIRED_TOKENS:
|
||||
check(tok in tokens, f"{where} does not list ${{{tok}}} in loadBalancerSourceRanges "
|
||||
"(R-DNS-03)")
|
||||
for r in ranges:
|
||||
m = TOKEN.match(str(r))
|
||||
if not m:
|
||||
assert_private(where, str(r))
|
||||
continue
|
||||
resolved = values.get(m.group(1))
|
||||
check(resolved, f"{where}: {r} is not defined in cluster-vars, so its value cannot be "
|
||||
"shown to be private (R-DNS-03)")
|
||||
for v in sorted(resolved or []):
|
||||
assert_private(f"{where} ({r})", v)
|
||||
|
||||
check(CHECKS >= 15, f"only {CHECKS} assertions ran — this check has gone vacuous (R-DNS-03)")
|
||||
|
||||
if FAILURES:
|
||||
for f in FAILURES:
|
||||
print(f"check FAIL: {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"check OK: pihole-dns-source-ranges — {CHECKS} assertions over "
|
||||
f"{len(frontends)} PiHole LoadBalancer Service(s)")
|
||||
PY
|
||||
@@ -0,0 +1,45 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — PiHole answers an explicit LAN name list, and no LAN-only name is left out of it
|
||||
# Catalog: docs/14 L0 row ("Pre-apply, in CI") — runs WITHOUT a cluster.
|
||||
#
|
||||
# Since PR #156 every public name is a proxied CNAME to the Cloudflare Tunnel and works from the
|
||||
# LAN, so PiHole's zone wildcard was replaced by an explicit list (catalog/pihole.yaml). That makes
|
||||
# the list load-bearing: a route disowned from public DNS and absent from it resolves nowhere. This
|
||||
# test derives the required names from the tree (every `controller: none` HTTPRoute, every hostname
|
||||
# the tunnel 404s, the L4 Services' own VIPs) and asserts the list covers them, carries no
|
||||
# zone-wide entry, and pins gitea and nextcloud to the Gateway VIP. It proves itself non-vacuous by
|
||||
# mutation. Rationale and the honest NOT-proven list: check.sh.
|
||||
#
|
||||
# WHY SCOPE, NOT REQUIREMENTS: the property spans catalog/, apps/, platform/ and runbooks/; a new
|
||||
# internal service in any of them can break it, and ci/touched-requirements.sh maps apps/* to no
|
||||
# requirement at all, so a requirement-scoped selection would skip it on precisely those changes.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: pihole-lan-names
|
||||
labels:
|
||||
conformance.home.k3s/R-DNS-03: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-DNS-03"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "true"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
PiHole answers an explicit list of LAN names rather than the whole zone; every HTTPRoute
|
||||
disowned from public DNS has an entry to the Gateway VIP; every hostname the Cloudflare
|
||||
Tunnel 404s has an entry, with the L4 names on their own VIPs; and gitea and nextcloud stay
|
||||
on the LAN path, off Cloudflare's request-size and timeout limits.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-pihole-explicit-lan-names
|
||||
description: >-
|
||||
Parse FTLCONF_misc_dnsmasq_lines from catalog/pihole.yaml and the rendered
|
||||
apps/wave-0/pihole.yaml, derive the required names from every `controller: none` HTTPRoute,
|
||||
the http_status:404 hostnames in platform/cloudflared/config.yaml and the L4 Services'
|
||||
lbipam annotations, assert the list covers them and contains no zone-wide entry, and run
|
||||
the evaluator on mutated lists to prove it is not vacuous.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — pihole-lan-names
|
||||
# Serves: R-DNS-03 (LAN split-horizon answers), and decisions/2026-11-09-pihole-explicit-lan-names.md.
|
||||
#
|
||||
# WHY THIS EXISTS. Since PR #156 every PUBLIC name is a proxied CNAME to the Cloudflare Tunnel and
|
||||
# works from the LAN, so PiHole no longer answers the whole zone with a wildcard: it answers an
|
||||
# EXPLICIT list (catalog/pihole.yaml, FTLCONF_misc_dnsmasq_lines) and forwards everything else to
|
||||
# the public resolvers. That inverts the old failure mode. With the wildcard, a forgotten name still
|
||||
# resolved; now a forgotten LAN-ONLY name — a route disowned from public DNS, which the tunnel also
|
||||
# 404s — resolves NOWHERE, and the app is silently unreachable from every client. Nothing in the
|
||||
# cluster turns red when that happens. So the list is derived from the tree and asserted here.
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (0) the dnsmasq value parses (only `local-ttl=` and `address=/<domain>/<${VAR}>` entries,
|
||||
# every VAR is a cluster-var), the catalog value and the rendered apps/wave-0/pihole.yaml
|
||||
# value are identical, and the inventories below are non-empty (anti-vacuous-pass);
|
||||
# (a) NO entry covers the whole zone: no `address=/${BASE_DOMAIN}/`, no `address=/#/`, and no
|
||||
# entry for any parent of the zone (dnsmasq matches every subdomain of an entry);
|
||||
# (b) every HTTPRoute hostname disowned from public DNS
|
||||
# (`external-dns.alpha.kubernetes.io/controller: none`, anywhere in apps/ platform/
|
||||
# runbooks/) has an explicit entry to ${GATEWAY_VIP} — a disowned route with no
|
||||
# `hostnames:` is itself a FAIL, because it cannot be checked;
|
||||
# (c) every hostname platform/cloudflared/config.yaml maps to http_status:404 has an entry;
|
||||
# the L4 DNS-only names (derived: Services carrying both
|
||||
# `external-dns.alpha.kubernetes.io/hostname` and `lbipam.cilium.io/ips` — unifi,
|
||||
# crafty-mc) must map to THAT Service's VIP, not the Gateway;
|
||||
# (d) gitea and nextcloud are pinned to ${GATEWAY_VIP} (Cloudflare's 100 MB body / 100 s
|
||||
# limits break git push/LFS and large uploads);
|
||||
# AND (e) the evaluator is not vacuous: it is run against mutated copies of the real entry list
|
||||
# (zone wildcard added; a disowned host dropped; a 404'd host dropped; gitea repointed;
|
||||
# unifi repointed to the Gateway) and must flag each one.
|
||||
#
|
||||
# NOT proven (static): that the running PiHole pods carry this value (FTL applies FTLCONF_* at
|
||||
# start; the env change rolls the StatefulSet), that public resolution of the unlisted names
|
||||
# reaches Cloudflare, or the substituted IPs. Runtime check: `dig @${PIHOLE_DNS_VIP} <name>`.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)" # repo root
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("CHECK FAIL (pihole-lan-names): PyYAML is required", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
FAILURES, CHECKS = [], 0
|
||||
NAME = "pihole-lan-names"
|
||||
ZONE = "${BASE_DOMAIN}"
|
||||
GW = "${GATEWAY_VIP}"
|
||||
DISOWN = "external-dns.alpha.kubernetes.io/controller"
|
||||
EDNS_HOST = "external-dns.alpha.kubernetes.io/hostname"
|
||||
LB_IPS = "lbipam.cilium.io/ips"
|
||||
# Public, but must bypass Cloudflare's 100 MB request body / 100 s response limits.
|
||||
LAN_PINNED = ("gitea", "nextcloud")
|
||||
# Trees that hold applied manifests. conformance/ fixtures and .claude/ worktrees are not authored
|
||||
# surface (CLAUDE.md §0); decisions/ is prose.
|
||||
SCAN = ("apps", "platform", "runbooks")
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def docs(path):
|
||||
try:
|
||||
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
||||
except yaml.YAMLError:
|
||||
return []
|
||||
|
||||
|
||||
def dnsmasq_value(objs):
|
||||
"""The FTLCONF_misc_dnsmasq_lines value from a list of docs (a ServiceDefinition or manifests)."""
|
||||
found = []
|
||||
for d in objs:
|
||||
envs = list(d.get("env") or [])
|
||||
tmpl = (((d.get("spec") or {}).get("template") or {}).get("spec") or {})
|
||||
for c in tmpl.get("containers") or []:
|
||||
envs += c.get("env") or []
|
||||
found += [e.get("value") for e in envs if e.get("name") == "FTLCONF_misc_dnsmasq_lines"]
|
||||
return found
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"CHECK FAIL ({NAME}): {msg}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
CATALOG = ROOT / "catalog" / "pihole.yaml"
|
||||
RENDERED = ROOT / "apps" / "wave-0" / "pihole.yaml"
|
||||
CF = ROOT / "platform" / "cloudflared" / "config.yaml"
|
||||
for p in (CATALOG, RENDERED, CF):
|
||||
if not p.is_file():
|
||||
die(f"{p.relative_to(ROOT)} is absent — the surface this check guards moved; re-point or "
|
||||
"delete this check in the SAME change")
|
||||
|
||||
cat_vals = dnsmasq_value(docs(CATALOG))
|
||||
ren_vals = dnsmasq_value(docs(RENDERED))
|
||||
check(len(cat_vals) == 1, f"(0) catalog/pihole.yaml declares FTLCONF_misc_dnsmasq_lines exactly "
|
||||
f"once (found {len(cat_vals)})")
|
||||
check(len(ren_vals) == 1, f"(0) apps/wave-0/pihole.yaml carries FTLCONF_misc_dnsmasq_lines exactly "
|
||||
f"once (found {len(ren_vals)})")
|
||||
if not cat_vals:
|
||||
die("no dnsmasq value to check")
|
||||
check(cat_vals[:1] == ren_vals[:1],
|
||||
"(0) the rendered apps/wave-0/pihole.yaml dnsmasq value differs from catalog/pihole.yaml — "
|
||||
"re-render pihole (catalog/render.py --no-image-automation catalog/pihole.yaml)")
|
||||
|
||||
cv = next(d for d in docs(ROOT / "clusters" / "base" / "cluster-vars.yaml")
|
||||
if d.get("kind") == "ConfigMap")
|
||||
CLUSTER_VARS = set(cv["data"])
|
||||
BASE_DOMAIN_VALUE = cv["data"]["BASE_DOMAIN"]
|
||||
|
||||
ENTRY = re.compile(r"^address=/([^/]+)/(\$\{[A-Z0-9_]+\})$")
|
||||
|
||||
|
||||
def parse(value):
|
||||
"""Return (entries {domain: target}, problems)."""
|
||||
entries, problems = {}, []
|
||||
for raw in value.split(";"):
|
||||
line = raw.strip()
|
||||
if raw != line:
|
||||
problems.append(f"entry {raw!r} has surrounding whitespace (FTL writes it verbatim)")
|
||||
if not line:
|
||||
problems.append("empty entry (doubled or trailing `;`)")
|
||||
continue
|
||||
if re.fullmatch(r"local-ttl=\d+", line):
|
||||
continue
|
||||
m = ENTRY.match(line)
|
||||
if not m:
|
||||
problems.append(f"entry {line!r} is not `local-ttl=N` or `address=/<domain>/${{VAR}}` — "
|
||||
f"this check cannot reason about it")
|
||||
continue
|
||||
dom, tgt = m.groups()
|
||||
if tgt[2:-1] not in CLUSTER_VARS:
|
||||
problems.append(f"{dom} -> {tgt}: not a key of clusters/base/cluster-vars.yaml")
|
||||
if dom in entries and entries[dom] != tgt:
|
||||
problems.append(f"{dom} is listed twice with different targets")
|
||||
entries[dom] = tgt
|
||||
return entries, problems
|
||||
|
||||
|
||||
entries, problems = parse(cat_vals[0])
|
||||
for pr in problems:
|
||||
check(False, f"(0) {pr}")
|
||||
|
||||
# ── inventories, derived from the tree ──────────────────────────────────────────────────────────
|
||||
disowned, unnamed = set(), []
|
||||
l4 = {}
|
||||
for top in SCAN:
|
||||
for f in sorted((ROOT / top).rglob("*.yaml")):
|
||||
for d in docs(f):
|
||||
meta = d.get("metadata") or {}
|
||||
ann = meta.get("annotations") or {}
|
||||
if d.get("kind") == "HTTPRoute" and str(ann.get(DISOWN, "")).strip() == "none":
|
||||
hosts = (d.get("spec") or {}).get("hostnames") or []
|
||||
if not hosts:
|
||||
unnamed.append(f"{f.relative_to(ROOT)}:{meta.get('name')}")
|
||||
disowned.update(hosts)
|
||||
if d.get("kind") == "Service" and EDNS_HOST in ann and LB_IPS in ann:
|
||||
for h in str(ann[EDNS_HOST]).split(","):
|
||||
l4[h.strip()] = str(ann[LB_IPS]).strip()
|
||||
for u in unnamed:
|
||||
check(False, f"(b) disowned HTTPRoute {u} declares no `hostnames:` — it inherits a listener "
|
||||
f"hostname this check cannot see; name it explicitly")
|
||||
check(len(disowned) >= 5, f"(0) only {len(disowned)} disowned HTTPRoute hostnames found — the scan "
|
||||
f"is vacuous (the disown annotation was renamed?)")
|
||||
check(len(l4) >= 2, f"(0) only {len(l4)} L4 Services publishing their own hostname found (expected "
|
||||
f"unifi and crafty-mc) — the scan is vacuous")
|
||||
|
||||
cf = yaml.safe_load(CF.read_text())
|
||||
cf_404 = {r["hostname"] for r in cf.get("ingress") or []
|
||||
if r.get("hostname") and r.get("service") == "http_status:404"}
|
||||
check(len(cf_404) >= 5, f"(0) only {len(cf_404)} http_status:404 hostnames in the cloudflared "
|
||||
f"config — the scan is vacuous")
|
||||
|
||||
|
||||
def covers_zone(dom):
|
||||
"""True if an entry for `dom` answers every name in the zone (dom is the zone or a parent)."""
|
||||
if dom in ("#", "", "."):
|
||||
return True
|
||||
if dom == ZONE:
|
||||
return True
|
||||
# A literal parent (e.g. `keithjackson.dev`, `dev`) of the zone's substituted value.
|
||||
real = dom.replace(ZONE, BASE_DOMAIN_VALUE)
|
||||
return real == BASE_DOMAIN_VALUE or BASE_DOMAIN_VALUE.endswith("." + real)
|
||||
|
||||
|
||||
def evaluate(entries):
|
||||
bad = []
|
||||
# (a)
|
||||
for dom in entries:
|
||||
if covers_zone(dom):
|
||||
bad.append(f"(a) `address=/{dom}/` answers EVERY name in the zone — the wildcard is gone "
|
||||
f"on purpose (public names must resolve to Cloudflare from the LAN)")
|
||||
# (b)
|
||||
for h in sorted(disowned):
|
||||
if entries.get(h) != GW:
|
||||
bad.append(f"(b) {h} is disowned from public DNS but PiHole maps it to "
|
||||
f"{entries.get(h, 'NOTHING')} — expected {GW}; without it the app is "
|
||||
f"unreachable everywhere")
|
||||
# (c)
|
||||
for h in sorted(cf_404):
|
||||
if h in l4:
|
||||
if entries.get(h) != l4[h]:
|
||||
bad.append(f"(c) {h} is an L4 DNS-only name; PiHole maps it to "
|
||||
f"{entries.get(h, 'NOTHING')}, expected its own VIP {l4[h]}")
|
||||
elif h not in entries:
|
||||
bad.append(f"(c) the tunnel 404s {h} but PiHole has no entry for it — it would resolve "
|
||||
f"only publicly, and the tunnel refuses it")
|
||||
# (d)
|
||||
for n in LAN_PINNED:
|
||||
h = f"{n}.{ZONE}"
|
||||
if entries.get(h) != GW:
|
||||
bad.append(f"(d) {h} maps to {entries.get(h, 'NOTHING (public/Cloudflare)')}; it must be "
|
||||
f"pinned to {GW} — Cloudflare caps request bodies at 100 MB and responses at "
|
||||
f"100 s")
|
||||
return bad
|
||||
|
||||
|
||||
for v in evaluate(entries):
|
||||
check(False, v)
|
||||
|
||||
# (e) mutation self-test
|
||||
mutants = {}
|
||||
m = dict(entries); m[ZONE] = GW
|
||||
mutants["zone wildcard added"] = m
|
||||
m = dict(entries); m[BASE_DOMAIN_VALUE.split(".", 1)[1]] = GW
|
||||
mutants["parent-of-zone wildcard added"] = m
|
||||
m = dict(entries); m.pop(sorted(disowned)[0], None)
|
||||
mutants[f"disowned host {sorted(disowned)[0]} dropped"] = m
|
||||
pure_404 = sorted(h for h in cf_404 if h not in l4 and h not in disowned)
|
||||
if pure_404:
|
||||
m = dict(entries); m.pop(pure_404[0], None)
|
||||
mutants[f"404'd host {pure_404[0]} dropped"] = m
|
||||
m = dict(entries); m.pop(f"gitea.{ZONE}", None)
|
||||
mutants["gitea unpinned"] = m
|
||||
m = dict(entries); m[f"nextcloud.{ZONE}"] = "${UNIFI_L4_VIP}"
|
||||
mutants["nextcloud repointed"] = m
|
||||
if l4:
|
||||
h = sorted(l4)[0]
|
||||
m = dict(entries); m[h] = GW
|
||||
mutants[f"{h} repointed to the Gateway"] = m
|
||||
for name, mut in mutants.items():
|
||||
check(evaluate(mut), f"(e) self-test: the evaluator did NOT flag the mutant '{name}' — vacuous")
|
||||
check(len(mutants) >= 5, f"(e) only {len(mutants)} mutants built — the self-test degenerated")
|
||||
_, pr = parse("local-ttl=60;address=/x.${BASE_DOMAIN}/${NOT_A_CLUSTER_VAR}; address=/y/${GATEWAY_VIP}")
|
||||
check(len(pr) >= 2, "(e) self-test: the parser accepted an unknown VAR or a whitespace-padded entry")
|
||||
|
||||
if CHECKS < 15:
|
||||
FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions; {len(entries)} PiHole names, {len(disowned)} disowned "
|
||||
f"routes, {len(cf_404)} tunnel-404 names, {len(l4)} L4 names")
|
||||
PY
|
||||
@@ -42,6 +42,19 @@
|
||||
# whose every solver is an automated challenge type (dns01 / http01) — the
|
||||
# same code path serves issuance AND renewal, so a solver a human must
|
||||
# complete would defeat both;
|
||||
# (5b) the ONE exception to (5) — WORKLOAD mTLS, added 2026-10-29 — is a Certificate
|
||||
# annotated `homelab.keithjackson.dev/tls-scope: workload-mtls`. The spec's TLS
|
||||
# requirements are about EXTERNALLY REACHABLE services (R-TLS-01; docs/08), and
|
||||
# ADR-0006 says outright that "mTLS between workloads is a separate concern … not
|
||||
# covered here". A public ACME CA cannot issue for a `*.svc.cluster.local` name or a
|
||||
# client-auth certificate at all. The annotation is NOT trusted: an exempt
|
||||
# Certificate must PROVE it is workload-internal — its Secret is served by NO Gateway
|
||||
# listener, every DNS name / commonName is cluster-internal (a bare label or
|
||||
# `*.svc[.cluster.local]`), it carries no IP/URI/email SANs, and its issuer is a
|
||||
# NAMESPACED Issuer in its own namespace whose spec is `selfSigned` or a `ca` backed
|
||||
# by a Secret that a tracked, likewise-exempt Certificate produces. Issuance and
|
||||
# renewal stay unattended (cert-manager renews CA-issued certs itself), and (9)
|
||||
# still applies. Every exemption is printed. First user: platform/buildkit.
|
||||
# (6) each dns01 solver's provider credential is referenced BY NAME and that
|
||||
# Secret is delivered SOPS-encrypted from git via a kustomization — the
|
||||
# operator's one-time token hand-off (docs/08 OQ-4), never a per-cert step;
|
||||
@@ -265,6 +278,7 @@ CERT_SECRETS: dict[tuple[str, str], str] = { # (namespace, secretName)
|
||||
for rel, d in CERTS
|
||||
}
|
||||
listeners_checked = 0
|
||||
GATEWAY_SECRETS: set[tuple[str, str]] = set() # every (namespace, Secret) a listener serves
|
||||
for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
|
||||
gw_ns = ns_of(gw)
|
||||
for lis in ((gw.get("spec") or {}).get("listeners") or []):
|
||||
@@ -280,6 +294,7 @@ for rel, gw in of_kind("Gateway", "gateway.networking.k8s.io/"):
|
||||
for r in refs:
|
||||
kind = str(r.get("kind", "Secret"))
|
||||
key = (str(r.get("namespace") or gw_ns), str(r.get("name")))
|
||||
GATEWAY_SECRETS.add(key)
|
||||
check(kind == "Secret",
|
||||
f"{rel}: listener {lis.get('name')!r} certificateRef kind is {kind!r}; only "
|
||||
"Secret refs can be traced to a cert-manager Certificate")
|
||||
@@ -324,8 +339,61 @@ CM_NS = ns_of(cm_releases[0][1], "cert-manager") if cm_releases else "cert-manag
|
||||
ISSUER_INDEX = {(d.get("kind"), name_of(d)): (rel, d) for rel, d in ISSUERS}
|
||||
solver_creds: list[tuple[str, str, str, str]] = [] # (issuer, ns, secretName, ref field)
|
||||
issuers_verified: set[str] = set()
|
||||
|
||||
# (5b) workload-mTLS exemption — see the header. Proven, never taken on trust.
|
||||
SCOPE_ANN = "homelab.keithjackson.dev/tls-scope"
|
||||
INTERNAL_NAME = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$")
|
||||
NS_ISSUERS = {(ns_of(d), name_of(d)): (rel, d) for rel, d in of_kind("Issuer", "cert-manager.io/")}
|
||||
EXEMPT_SECRETS = {(ns_of(d), str((d.get("spec") or {}).get("secretName", "")))
|
||||
for _, d in CERTS
|
||||
if ((d.get("metadata") or {}).get("annotations") or {}).get(SCOPE_ANN) == "workload-mtls"}
|
||||
exempted: list[str] = []
|
||||
|
||||
def workload_mtls_ok(rel: str, cert: dict) -> bool:
|
||||
"""True if `cert` claims the workload-mTLS scope AND proves it; records failures."""
|
||||
ann = ((cert.get("metadata") or {}).get("annotations") or {}).get(SCOPE_ANN)
|
||||
if ann is None:
|
||||
return False
|
||||
cns, cname = ns_of(cert), name_of(cert)
|
||||
spec = cert.get("spec") or {}
|
||||
ok = True
|
||||
def need(cond, msg):
|
||||
nonlocal ok
|
||||
check(cond, f"{rel}: Certificate {cns}/{cname} claims {SCOPE_ANN}={ann!r} but {msg}")
|
||||
ok = ok and bool(cond)
|
||||
need(ann == "workload-mtls", "the only accepted value is 'workload-mtls'")
|
||||
need((cns, str(spec.get("secretName", ""))) not in GATEWAY_SECRETS,
|
||||
"its Secret is served by a Gateway listener — that is external TLS, which must be ACME (R-TLS-01)")
|
||||
names = [str(n) for n in (spec.get("dnsNames") or [])]
|
||||
if spec.get("commonName"):
|
||||
names.append(str(spec["commonName"]))
|
||||
for n in names:
|
||||
internal = INTERNAL_NAME.match(n) and ("." not in n or n.endswith(".svc") or n.endswith(".svc.cluster.local"))
|
||||
need(bool(internal), f"name {n!r} is not cluster-internal (bare label or *.svc[.cluster.local])")
|
||||
for f in ("ipAddresses", "uris", "emailAddresses"):
|
||||
need(not spec.get(f), f"it carries {f}, which a workload-internal certificate has no use for")
|
||||
ref = spec.get("issuerRef") or {}
|
||||
need(str(ref.get("kind", "Issuer")) == "Issuer",
|
||||
"its issuer is not a NAMESPACED Issuer — a ClusterIssuer could sign for any namespace")
|
||||
hit = NS_ISSUERS.get((cns, str(ref.get("name", ""))))
|
||||
need(hit is not None, f"its Issuer {ref.get('name')!r} is not tracked in namespace {cns!r}")
|
||||
if hit is not None:
|
||||
ispec = hit[1].get("spec") or {}
|
||||
kinds = sorted(ispec)
|
||||
need(kinds in (["selfSigned"], ["ca"]),
|
||||
f"its Issuer is {kinds} — only selfSigned or ca are accepted for workload mTLS")
|
||||
if kinds == ["ca"]:
|
||||
need((cns, str((ispec.get("ca") or {}).get("secretName", ""))) in EXEMPT_SECRETS,
|
||||
"its CA Issuer's Secret is not produced by a tracked workload-mtls Certificate "
|
||||
"in the same namespace — the CA itself would have to be minted by hand")
|
||||
if ok:
|
||||
exempted.append(f"{cns}/{cname}")
|
||||
return ok
|
||||
|
||||
for rel, cert in CERTS:
|
||||
spec = cert.get("spec") or {}
|
||||
if workload_mtls_ok(rel, cert):
|
||||
continue # (5b): proven workload-internal; (9) below still applies
|
||||
ref = spec.get("issuerRef") or {}
|
||||
kind = str(ref.get("kind", "Issuer"))
|
||||
per_env = resolve_per_env(str(ref.get("name", "")))
|
||||
@@ -379,6 +447,8 @@ for rel, cert in CERTS:
|
||||
solver_creds.append((iname, want_ns, n, f"{provider}.{field}"))
|
||||
|
||||
check(len(issuers_verified) > 0, "no issuer was reachable from any Certificate — (5) proved nothing")
|
||||
if exempted:
|
||||
print(f" (5b) workload-mTLS certificates exempted from ACME after proving it: {', '.join(exempted)}")
|
||||
|
||||
# (6) the solver credential is delivered, SOPS-encrypted, from git — never per-cert by hand.
|
||||
check(len(solver_creds) > 0,
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — accepted CVE risk has a reason and a bounded expiry (R-VULN-08, docs/17 "Risk acceptance").
|
||||
# Offline, no cluster.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-acceptances-schema
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-08: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-08"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
platform/vuln-scanner/acceptances.yaml rejects any entry without a reason or expiry, with a
|
||||
malformed expiry, or expiring more than 180 days out. Proven with mutated negative fixtures.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-acceptance-schema
|
||||
description: >-
|
||||
Validate acceptances.yaml against the docs/17 shape, then run the validator on mutated
|
||||
negative fixtures to prove it can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-acceptances-schema
|
||||
# Serves: R-VULN-08 (docs/17 "Risk acceptance": "An entry with no reason or expiry is rejected by the gate").
|
||||
#
|
||||
# GIVEN platform/vuln-scanner/acceptances.yaml,
|
||||
# THEN (1) it is a YAML list;
|
||||
# (2) every entry has a non-empty id, a non-empty reason, an expires of the form YYYY-MM-DD
|
||||
# at most 180 days from today, and only the permitted keys (image is optional);
|
||||
# AND (3) the validator rejects mutated negative fixtures for each rule and accepts a good entry.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import datetime as dt, re, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-acceptances-schema"
|
||||
FAILURES, CHECKS = [], 0
|
||||
TODAY = dt.datetime.now(dt.timezone.utc).date()
|
||||
MAX_DAYS = 180
|
||||
ALLOWED_KEYS = {"id", "image", "reason", "expires"}
|
||||
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def validate(entries):
|
||||
errs = []
|
||||
if not isinstance(entries, list):
|
||||
return ["acceptances must be a YAML list"]
|
||||
for i, e in enumerate(entries):
|
||||
if not isinstance(e, dict):
|
||||
errs.append(f"entry {i} is not a mapping")
|
||||
continue
|
||||
if set(e) - ALLOWED_KEYS:
|
||||
errs.append(f"entry {i} has unknown keys {sorted(set(e) - ALLOWED_KEYS)}")
|
||||
for field in ("id", "reason"):
|
||||
if not isinstance(e.get(field), str) or not e[field].strip():
|
||||
errs.append(f"entry {i} has no non-empty {field}")
|
||||
if "image" in e and (not isinstance(e["image"], str) or not e["image"].strip()):
|
||||
errs.append(f"entry {i} has an empty image")
|
||||
exp = e.get("expires")
|
||||
if isinstance(exp, dt.date) and not isinstance(exp, dt.datetime):
|
||||
exp_s = exp.isoformat()
|
||||
elif isinstance(exp, str):
|
||||
exp_s = exp
|
||||
else:
|
||||
errs.append(f"entry {i} has no expires")
|
||||
continue
|
||||
if not DATE_RE.match(exp_s):
|
||||
errs.append(f"entry {i} expires is malformed: {exp_s!r}")
|
||||
continue
|
||||
try:
|
||||
d = dt.date.fromisoformat(exp_s)
|
||||
except ValueError:
|
||||
errs.append(f"entry {i} expires is not a real date: {exp_s!r}")
|
||||
continue
|
||||
if (d - TODAY).days > MAX_DAYS:
|
||||
errs.append(f"entry {i} expires {exp_s}, more than {MAX_DAYS} days out")
|
||||
return errs
|
||||
|
||||
|
||||
path = ROOT / "platform" / "vuln-scanner" / "acceptances.yaml"
|
||||
if not path.is_file():
|
||||
print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/acceptances.yaml is absent", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
entries = yaml.safe_load(path.read_text())
|
||||
check(entries == [] or (isinstance(entries, list) and entries), "(1) acceptances.yaml parsed to a list")
|
||||
errs = validate(entries)
|
||||
check(not errs, f"(2) acceptances.yaml is valid: {errs}")
|
||||
|
||||
soon = (TODAY + dt.timedelta(days=30)).isoformat()
|
||||
far = (TODAY + dt.timedelta(days=200)).isoformat()
|
||||
good = {"id": "CVE-2026-00000", "image": "docker.io/library/x", "reason": "not reachable", "expires": soon}
|
||||
check(validate([good]) == [], "(3) a well-formed entry is accepted")
|
||||
check(validate([{k: v for k, v in good.items() if k != "image"}]) == [], "(3) image is optional (platform-wide)")
|
||||
for name, entry in {
|
||||
"no reason": {k: v for k, v in good.items() if k != "reason"},
|
||||
"empty reason": dict(good, reason=""),
|
||||
"no expires": {k: v for k, v in good.items() if k != "expires"},
|
||||
"no id": {k: v for k, v in good.items() if k != "id"},
|
||||
"malformed expiry": dict(good, expires="next-tuesday"),
|
||||
"impossible date": dict(good, expires="2027-13-45"),
|
||||
"past 180 days": dict(good, expires=far),
|
||||
"unknown key": dict(good, owner="me"),
|
||||
}.items():
|
||||
check(bool(validate([entry])), f"(3) the validator accepted a bad entry: {name}")
|
||||
check(bool(validate({"not": "a list"})), "(3) the validator accepted a non-list")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -0,0 +1,29 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the five scanner alerts exist and are unit-tested, including the absent-metric cases
|
||||
# (R-VULN-06, docs/17 "Acceptance gates"). Offline, no cluster.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-alerts-unit-tested
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-06"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
VulnScanStale, VulnDatabaseStale, VulnCoverageGap, VulnCriticalFixAvailable and
|
||||
VulnTriageBotSilent exist, each promtool-tested firing, and the two absent-metric rules fire
|
||||
in a test with no such series. Proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-alert-unit-tests
|
||||
description: >-
|
||||
Parse alerts.yaml and alerts.promtool-test.yaml, assert every alert and the two absent cases
|
||||
are pinned, run the harness when promtool is present, and run the evaluator on mutated copies.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-alerts-unit-tested
|
||||
# Serves: R-VULN-06 (docs/17 "Acceptance gates": alerts unit-tested, including the absent-metric cases).
|
||||
#
|
||||
# GIVEN platform/vuln-scanner/alerts.yaml and alerts.promtool-test.yaml,
|
||||
# THEN (1) the rules are exactly the five contract alerts, with the two absent()-bearing rules;
|
||||
# (2) each alert is asserted FIRING in a promtool test;
|
||||
# (3) VulnScanStale and VulnTriageBotSilent each fire in a test with NO input series for their
|
||||
# own metric (the absent case) and in one where it vanishes (re-fire);
|
||||
# (4) each alert carries severity and a runbook_url;
|
||||
# (5) when promtool is present (the toolbox), the actual harness passes on this file;
|
||||
# AND (6) the evaluator is run on mutated copies (the absent arm deleted, a firing assertion
|
||||
# removed, an alert dropped) and must flag each (non-vacuity).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy, re, shutil, subprocess, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-alerts-unit-tested"
|
||||
FAILURES, CHECKS = [], 0
|
||||
ALERTS = {"VulnScanStale", "VulnDatabaseStale", "VulnCoverageGap", "VulnCriticalFixAvailable", "VulnTriageBotSilent"}
|
||||
ABSENT_ALERTS = {
|
||||
"VulnScanStale": "vuln_scanner_last_complete_run_timestamp_seconds",
|
||||
"VulnTriageBotSilent": "vuln_scanner_triage_heartbeat_timestamp_seconds",
|
||||
}
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
COMP = ROOT / "platform" / "vuln-scanner"
|
||||
for f in ("alerts.yaml", "alerts.promtool-test.yaml"):
|
||||
if not (COMP / f).is_file():
|
||||
print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/{f} is absent", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
rules = yaml.safe_load((COMP / "alerts.yaml").read_text())
|
||||
tests = yaml.safe_load((COMP / "alerts.promtool-test.yaml").read_text())
|
||||
|
||||
|
||||
def alert_rules(doc):
|
||||
return {r["alert"]: r for g in doc["spec"]["groups"] for r in g["rules"]}
|
||||
|
||||
|
||||
def eval_alerts(doc):
|
||||
bad = []
|
||||
got = alert_rules(doc)
|
||||
if set(got) != ALERTS:
|
||||
bad.append(f"alerts are {sorted(got)}, expected {sorted(ALERTS)}")
|
||||
for name, metric in ABSENT_ALERTS.items():
|
||||
if name in got and f"absent({metric})" not in got[name]["expr"].replace(" ", ""):
|
||||
bad.append(f"{name} lacks the absent({metric}) arm")
|
||||
for name, r in got.items():
|
||||
if not (r.get("labels") or {}).get("severity"):
|
||||
bad.append(f"{name} has no severity label")
|
||||
if "/runbooks/" not in ((r.get("annotations") or {}).get("runbook_url") or ""):
|
||||
bad.append(f"{name} has no runbook_url")
|
||||
return bad
|
||||
|
||||
|
||||
def firing_assertions(test):
|
||||
"""(alertname -> assertion list) for the firing expectations of one promtool test."""
|
||||
out = {}
|
||||
for pe in test.get("promql_expr_test") or []:
|
||||
expr, samples = pe.get("expr", ""), pe.get("exp_samples") or []
|
||||
m = re.search(r'alertname="(Vuln\w+)"', expr)
|
||||
if m and samples and 'alertstate="firing"' in (expr + " " + " ".join(s.get("labels", "") for s in samples)):
|
||||
out.setdefault(m.group(1), []).append(pe)
|
||||
return out
|
||||
|
||||
|
||||
def series_for(test):
|
||||
return [s.get("series", "") for s in test.get("input_series") or []]
|
||||
|
||||
|
||||
def eval_tests(doc):
|
||||
bad = []
|
||||
all_tests = doc.get("tests") or []
|
||||
fired = set()
|
||||
for t in all_tests:
|
||||
fired |= set(firing_assertions(t))
|
||||
missing = ALERTS - fired
|
||||
if missing:
|
||||
bad.append(f"no firing promtool assertion for {sorted(missing)}")
|
||||
for alert, metric in ABSENT_ALERTS.items():
|
||||
absent_case = any(
|
||||
alert in firing_assertions(t) and not any(s.startswith(metric) for s in series_for(t))
|
||||
for t in all_tests)
|
||||
if not absent_case:
|
||||
bad.append(f"{alert} has no firing test with NO {metric} series (the absent case)")
|
||||
vanish_case = any(
|
||||
alert in firing_assertions(t) and any(s.startswith(metric) and "stale" in str(i.get("values", ""))
|
||||
for i in t.get("input_series", []) for s in [i.get("series", "")])
|
||||
for t in all_tests)
|
||||
if not vanish_case:
|
||||
bad.append(f"{alert} has no test where its metric goes stale (the re-fire case)")
|
||||
return bad
|
||||
|
||||
|
||||
for v in eval_alerts(rules):
|
||||
check(False, f"(1,4) {v}")
|
||||
for v in eval_tests(tests):
|
||||
check(False, f"(2,3) {v}")
|
||||
check(len(alert_rules(rules)) == 5, "(1) five alert rules parsed")
|
||||
|
||||
# (5) the real harness, only when the rule engine is present.
|
||||
if shutil.which("promtool"):
|
||||
rc = subprocess.run(["bash", str(ROOT / "platform/monitoring/tests/run.sh"),
|
||||
"platform/vuln-scanner/alerts.promtool-test.yaml"],
|
||||
cwd=str(ROOT), capture_output=True, text=True)
|
||||
check(rc.returncode == 0, f"(5) the promtool harness failed: rc={rc.returncode}\n{rc.stdout[-800:]}{rc.stderr[-800:]}")
|
||||
|
||||
# (6) non-vacuity
|
||||
m = copy.deepcopy(rules)
|
||||
for g in m["spec"]["groups"]:
|
||||
for r in g["rules"]:
|
||||
if r["alert"] == "VulnTriageBotSilent":
|
||||
r["expr"] = r["expr"].split(" or ")[0]
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: absent arm deleted")
|
||||
m = copy.deepcopy(rules)
|
||||
m["spec"]["groups"][0]["rules"].pop(1)
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: alert dropped")
|
||||
m = copy.deepcopy(rules)
|
||||
for g in m["spec"]["groups"]:
|
||||
for r in g["rules"]:
|
||||
r["annotations"].pop("runbook_url", None)
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: runbook_url removed")
|
||||
|
||||
m = copy.deepcopy(tests)
|
||||
for t in m["tests"]:
|
||||
t["promql_expr_test"] = [p for p in t["promql_expr_test"]
|
||||
if 'VulnCoverageGap' not in p.get("expr", "")]
|
||||
check(bool(eval_tests(m)), "(6) evaluator missed: firing assertion removed")
|
||||
m = copy.deepcopy(tests)
|
||||
m["tests"] = [t for t in m["tests"] if "ABSENT" not in t.get("name", "")]
|
||||
check(bool(eval_tests(m)), "(6) evaluator missed: absent-case tests removed")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -0,0 +1,28 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — Prometheus receives aggregates only (R-VULN-06, docs/17 "Alerting").
|
||||
# Offline, no cluster. Depends on platform/vuln-scanner/src/metrics.py (the SCANNER's file).
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-metrics-no-per-cve-label
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-06"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
No vuln_scanner_* metric or alert rule carries a label other than state, severity or fixable,
|
||||
so Prometheus can never hold a per-CVE, per-image or per-workload series. Proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-metric-labels-are-aggregates
|
||||
description: >-
|
||||
Extract every vuln_scanner_* metric label from metrics.py and alerts.yaml, assert the
|
||||
allowlist, and run the extractor on mutated text to prove it can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-metrics-no-per-cve-label
|
||||
# Serves: R-VULN-06 ("Prometheus receives aggregates only ... MUST NOT receive a per-CVE series").
|
||||
#
|
||||
# GIVEN platform/vuln-scanner/src/metrics.py and platform/vuln-scanner/alerts.yaml,
|
||||
# THEN (1) every label name attached to a vuln_scanner_* metric is in the allowlist
|
||||
# (state, severity, fixable);
|
||||
# (2) metrics.py exposes every metric name from the contract;
|
||||
# (3) alerts.yaml selects vuln_scanner_* metrics with no label matcher outside the allowlist;
|
||||
# AND (4) the extractor is proven on mutated text: a `cve=` label, a `{image=...}` selector and a
|
||||
# per-workload label are each flagged, and label-free text is not.
|
||||
# A missing metrics.py is a FAIL: this is a whole-repo-scoped check and the file is part of the same
|
||||
# delivery (the check has no skip path).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import re, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-metrics-no-per-cve-label"
|
||||
FAILURES, CHECKS = [], 0
|
||||
ALLOWED = {"state", "severity", "fixable"}
|
||||
METRICS = (
|
||||
"vuln_scanner_report_generated_timestamp_seconds",
|
||||
"vuln_scanner_last_complete_run_timestamp_seconds",
|
||||
"vuln_scanner_database_updated_timestamp_seconds",
|
||||
"vuln_scanner_targets",
|
||||
"vuln_scanner_findings",
|
||||
"vuln_scanner_failed_targets_over_24h",
|
||||
"vuln_scanner_critical_fix_overdue",
|
||||
"vuln_scanner_triage_heartbeat_timestamp_seconds",
|
||||
)
|
||||
# Label keys that could only carry per-CVE or per-target identity, anywhere a metric could be
|
||||
# written. Catches label construction the name{...} scan cannot see (e.g. "cve_id": cve_id).
|
||||
IDENTITY_LABEL = re.compile(r'''["'](cve|cve_id|advisory|package|pkg|image|digest|workload|namespace|pod|container|target|target_id)["']\s*[:=]''')
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def label_names(text):
|
||||
"""Every label key in `vuln_scanner_x{key="value",...}` occurrences (Python or PromQL text),
|
||||
tolerating f-strings' doubled braces and multi-line calls."""
|
||||
found = set()
|
||||
for m in re.finditer(r"vuln_scanner_[a-z0-9_]+\{", text):
|
||||
tail = text[m.end(): m.end() + 200]
|
||||
close = tail.find("}\n")
|
||||
chunk = tail[: close if close != -1 else None].split("\n", 1)[0]
|
||||
for key in re.findall(r"([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*\\?[\"'{]", chunk):
|
||||
found.add(key)
|
||||
return found
|
||||
|
||||
|
||||
METRICS_PY = ROOT / "platform" / "vuln-scanner" / "src" / "metrics.py"
|
||||
ALERTS = ROOT / "platform" / "vuln-scanner" / "alerts.yaml"
|
||||
if not METRICS_PY.is_file():
|
||||
print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/src/metrics.py is absent (the SCANNER's "
|
||||
f"deliverable); the aggregate-only property cannot be asserted without it", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
src = METRICS_PY.read_text()
|
||||
check(all(m in src for m in METRICS), f"(2) metrics.py names every contract metric "
|
||||
f"(missing: {[m for m in METRICS if m not in src]})")
|
||||
|
||||
labels = label_names(src)
|
||||
extra = labels - ALLOWED
|
||||
check(not extra, f"(1) metrics.py attaches labels outside the allowlist: {sorted(extra)}")
|
||||
hits = IDENTITY_LABEL.findall(src)
|
||||
check(not hits, f"(1) metrics.py constructs an identity-bearing metric label: {sorted(set(hits))}")
|
||||
|
||||
rules = yaml.safe_load(ALERTS.read_text())
|
||||
exprs = [r["expr"] for g in rules["spec"]["groups"] for r in g["rules"]]
|
||||
check(len(exprs) >= 5, "(3) alerts.yaml parsed five or more rules")
|
||||
alert_labels = set()
|
||||
for e in exprs:
|
||||
alert_labels |= label_names(e)
|
||||
check(not (alert_labels - ALLOWED),
|
||||
f"(3) alerts.yaml selects labels outside the allowlist: {sorted(alert_labels - ALLOWED)}")
|
||||
|
||||
# (4) non-vacuity
|
||||
negatives = {
|
||||
"cve label": 'vuln_scanner_findings{severity="HIGH",cve="CVE-1"} 1\n',
|
||||
"image selector": 'max(vuln_scanner_findings{image="docker.io/x"})\n',
|
||||
"workload label": 'f"vuln_scanner_targets{{workload=\\"{w}\\",state=\\"x\\"}} 1"\n',
|
||||
}
|
||||
for name, text in negatives.items():
|
||||
check(bool(label_names(text) - ALLOWED), f"(4) the extractor missed: {name}")
|
||||
ok_text = 'vuln_scanner_targets{state="scanned"} 3\nvuln_scanner_findings{severity="LOW",fixable="true"} 1\n'
|
||||
check(not (label_names(ok_text) - ALLOWED), "(4) the extractor flagged the allowed labels")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -0,0 +1,30 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the scanner's Flux tier cannot stall the platform, and its network reach is exactly what
|
||||
# docs/17 "Security of the scanner itself" allows (R-VULN-09). Offline, no cluster.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-isolation
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-09: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-09"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
The vuln-scanner Flux tier is wait:false, in neither gating tier, and nothing dependsOn it;
|
||||
ingress is exactly the Gateway to 4180, the CI runner to 8081 and Prometheus to 9090;
|
||||
egress reaches the API, DNS, the public internet on 443 only with cluster and LAN excluded,
|
||||
and the Keycloak path the sidecar needs. Proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-isolation-and-network-shape
|
||||
description: >-
|
||||
Parse the Flux wiring and networkpolicy.yaml, assert isolation and the exact network shape,
|
||||
and run the evaluators on mutated copies to prove they can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-scanner-isolation
|
||||
# Serves: R-VULN-09 (docs/17 "Security of the scanner itself").
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) the Flux tier is wait:false, dependsOn platform-config, in neither gating tier, nothing
|
||||
# anywhere dependsOn it, and both overlays carry its SOPS decryption patch;
|
||||
# (2) ingress is EXACTLY: reserved:ingress to 4180, act-runner to 8081, prometheus to 9090;
|
||||
# (3) egress to the world is tcp/443 only through a CIDR rule excluding every private range,
|
||||
# and the only other destinations are the API entity, the Gateway entity on 443 and Keycloak
|
||||
# (the oauth2-proxy sidecar's OIDC path, the one deliberate addition to the spec's list);
|
||||
# no `world`, `all`, `cluster`, `host` or `remote-node` entity;
|
||||
# (4) the namespace ships a default-deny baseline and is not app-namespace=true (single owner);
|
||||
# AND (5) each evaluator is run on a mutated copy and must flag it (non-vacuity).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy, re, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-scanner-isolation"
|
||||
FAILURES, CHECKS = [], 0
|
||||
FORBIDDEN_ENTITIES = {"world", "all", "cluster", "host", "remote-node"}
|
||||
PRIVATE_RANGES = {"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"}
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def docs(path):
|
||||
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
||||
|
||||
|
||||
fk_path = ROOT / "clusters" / "base" / "vuln-scanner.yaml"
|
||||
np_path = ROOT / "platform" / "vuln-scanner" / "networkpolicy.yaml"
|
||||
for p in (fk_path, np_path):
|
||||
if not p.is_file():
|
||||
print(f"CHECK FAIL ({NAME}): {p.relative_to(ROOT)} is absent", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
# (1) Flux wiring
|
||||
fk = next(d for d in docs(fk_path) if d.get("kind") == "Kustomization")
|
||||
check(fk["spec"].get("wait") is False, "(1) clusters/base/vuln-scanner.yaml must be wait: false")
|
||||
check([d.get("name") for d in fk["spec"].get("dependsOn") or []] == ["platform-config"],
|
||||
"(1) the tier dependsOn exactly platform-config")
|
||||
check(fk["spec"].get("path") == "./platform/vuln-scanner", "(1) the tier reconciles ./platform/vuln-scanner")
|
||||
check((fk["spec"].get("postBuild") or {}).get("substituteFrom"),
|
||||
"(1) postBuild.substituteFrom is present (the route/alerts use ${BASE_DOMAIN})")
|
||||
check("vuln-scanner.yaml" in (ROOT / "clusters" / "base" / "kustomization.yaml").read_text(),
|
||||
"(1) clusters/base/kustomization.yaml lists vuln-scanner.yaml")
|
||||
check("vuln-scanner" not in (ROOT / "platform" / "kustomization.yaml").read_text(),
|
||||
"(1) platform/kustomization.yaml must not include vuln-scanner (the gating tier)")
|
||||
for f in sorted((ROOT / "clusters").rglob("*.yaml")):
|
||||
for d in docs(f):
|
||||
for dep in (d.get("spec") or {}).get("dependsOn") or []:
|
||||
check(not (isinstance(dep, dict) and dep.get("name") == "vuln-scanner"),
|
||||
f"(1) {f.relative_to(ROOT)} {d.get('metadata', {}).get('name')} dependsOn vuln-scanner — "
|
||||
"a broken scanner must never stall anything")
|
||||
for env in ("dev", "prod"):
|
||||
text = (ROOT / "clusters" / env / "kustomization.yaml").read_text()
|
||||
check(re.search(r"name:\s*vuln-scanner\s*\n\s*namespace:\s*flux-system\s*\n\s*spec:\s*\n\s*decryption:",
|
||||
text) is not None,
|
||||
f"(1) clusters/{env}/kustomization.yaml carries the vuln-scanner SOPS decryption patch")
|
||||
|
||||
# Network shape
|
||||
np_docs = docs(np_path)
|
||||
dd = [d for d in np_docs if d.get("kind") == "NetworkPolicy" and d["metadata"]["name"] == "default-deny"]
|
||||
check(len(dd) == 1 and set(dd[0]["spec"]["policyTypes"]) == {"Ingress", "Egress"} and dd[0]["spec"].get("ingress") == [],
|
||||
"(4) default-deny baseline denies both directions with no ingress")
|
||||
ns = next(d for d in docs(ROOT / "platform" / "vuln-scanner" / "namespace.yaml") if d.get("kind") == "Namespace")
|
||||
check("homelab.keithjackson.dev/app-namespace" not in (ns["metadata"].get("labels") or {}),
|
||||
"(4) the namespace must not be app-namespace=true (the default-deny would be double-owned)")
|
||||
|
||||
|
||||
def cnp(name):
|
||||
return next((d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy" and d["metadata"]["name"] == name), None)
|
||||
|
||||
|
||||
def ports_of(rule):
|
||||
return {(p.get("port"), p.get("protocol", "TCP")) for tp in rule.get("toPorts") or [] for p in tp.get("ports") or []}
|
||||
|
||||
|
||||
def eval_ingress(policy):
|
||||
bad = []
|
||||
rules = policy["spec"].get("ingress") or []
|
||||
if len(rules) != 3:
|
||||
bad.append(f"expected exactly 3 ingress rules, found {len(rules)}")
|
||||
seen = set()
|
||||
for r in rules:
|
||||
ports = ports_of(r)
|
||||
if r.get("fromEntities") == ["ingress"] and ports == {("4180", "TCP")}:
|
||||
seen.add("gateway")
|
||||
elif len(r.get("fromEndpoints") or []) == 1:
|
||||
sel = r["fromEndpoints"][0].get("matchLabels") or {}
|
||||
if (sel.get("io.kubernetes.pod.namespace") == "gitea-actions"
|
||||
and sel.get("app.kubernetes.io/name") == "act-runner" and ports == {("8081", "TCP")}):
|
||||
seen.add("ci")
|
||||
elif (sel.get("io.kubernetes.pod.namespace") == "monitoring"
|
||||
and sel.get("app.kubernetes.io/name") == "prometheus" and ports == {("9090", "TCP")}):
|
||||
seen.add("prometheus")
|
||||
else:
|
||||
bad.append(f"unexpected ingress rule {r}")
|
||||
else:
|
||||
bad.append(f"unexpected ingress rule {r}")
|
||||
missing = {"gateway", "ci", "prometheus"} - seen
|
||||
if missing:
|
||||
bad.append(f"ingress paths missing: {sorted(missing)}")
|
||||
return bad
|
||||
|
||||
|
||||
def eval_egress(policy):
|
||||
bad = []
|
||||
rules = policy["spec"].get("egress") or []
|
||||
if len(rules) != 4:
|
||||
bad.append(f"expected exactly 4 egress rules, found {len(rules)}")
|
||||
world = 0
|
||||
for r in rules:
|
||||
ents = set(r.get("toEntities") or [])
|
||||
if ents & FORBIDDEN_ENTITIES:
|
||||
bad.append(f"egress admits forbidden entity {sorted(ents & FORBIDDEN_ENTITIES)}")
|
||||
if ents == {"kube-apiserver"} and not r.get("toPorts"):
|
||||
continue
|
||||
if ents == {"ingress"} and ports_of(r) == {("443", "TCP")}:
|
||||
continue
|
||||
if r.get("toEndpoints"):
|
||||
sel = r["toEndpoints"][0].get("matchLabels") or {}
|
||||
if not (len(r["toEndpoints"]) == 1 and sel.get("io.kubernetes.pod.namespace") == "keycloak"
|
||||
and sel.get("app") == "keycloak"):
|
||||
bad.append(f"egress to unexpected endpoints {r}")
|
||||
continue
|
||||
if r.get("toCIDRSet"):
|
||||
world += 1
|
||||
sets = r["toCIDRSet"]
|
||||
if len(sets) != 1 or sets[0].get("cidr") != "0.0.0.0/0":
|
||||
bad.append(f"unexpected CIDR rule {r}")
|
||||
continue
|
||||
if not PRIVATE_RANGES.issubset(set(sets[0].get("except") or [])):
|
||||
bad.append(f"the world CIDR rule does not exclude {sorted(PRIVATE_RANGES - set(sets[0].get('except') or []))}")
|
||||
if ports_of(r) != {("443", "TCP")}:
|
||||
bad.append(f"the world CIDR rule is not tcp/443 only ({sorted(ports_of(r))})")
|
||||
continue
|
||||
if r.get("toCIDR"):
|
||||
bad.append(f"egress names a bare CIDR rule {r}")
|
||||
continue
|
||||
if not ents:
|
||||
bad.append(f"unexpected egress rule {r}")
|
||||
if world != 1:
|
||||
bad.append(f"expected exactly one world CIDR rule, found {world}")
|
||||
return bad
|
||||
|
||||
|
||||
ing, eg = cnp("vuln-scanner-allow-ingress"), cnp("vuln-scanner-allow-egress")
|
||||
check(ing is not None, "(2) the ingress CiliumNetworkPolicy exists")
|
||||
check(eg is not None, "(3) the egress CiliumNetworkPolicy exists")
|
||||
if ing is not None:
|
||||
for v in eval_ingress(ing):
|
||||
check(False, f"(2) {v}")
|
||||
if eg is not None:
|
||||
for v in eval_egress(eg):
|
||||
check(False, f"(3) {v}")
|
||||
check(len([d for d in np_docs if d.get("kind") == "CiliumNetworkPolicy"]) == 2,
|
||||
"(3) exactly two CiliumNetworkPolicies, so no other path is added silently")
|
||||
|
||||
# (5) non-vacuity
|
||||
if ing is not None and eg is not None:
|
||||
def m_ing_open(p): p["spec"]["ingress"].append({"fromEntities": ["world"], "toPorts": [{"ports": [{"port": "8081", "protocol": "TCP"}]}]})
|
||||
def m_ing_ns(p): p["spec"]["ingress"][1]["fromEndpoints"][0]["matchLabels"]["io.kubernetes.pod.namespace"] = "default"
|
||||
def m_ing_port(p): p["spec"]["ingress"][0]["toPorts"][0]["ports"][0]["port"] = "8080"
|
||||
for name, fn in (("extra world ingress", m_ing_open), ("CI rule moved to another namespace", m_ing_ns),
|
||||
("Gateway reaches the page port", m_ing_port)):
|
||||
m = copy.deepcopy(ing)
|
||||
fn(m)
|
||||
check(bool(eval_ingress(m)), f"(5) ingress evaluator missed mutant: {name}")
|
||||
def m_eg_world(p): p["spec"]["egress"].append({"toEntities": ["world"]})
|
||||
def m_eg_lan(p):
|
||||
for r in p["spec"]["egress"]:
|
||||
if r.get("toCIDRSet"):
|
||||
r["toCIDRSet"][0]["except"] = ["10.0.0.0/8"]
|
||||
def m_eg_port(p):
|
||||
for r in p["spec"]["egress"]:
|
||||
if r.get("toCIDRSet"):
|
||||
r["toPorts"][0]["ports"].append({"port": "80", "protocol": "TCP"})
|
||||
def m_eg_endpoint(p):
|
||||
for r in p["spec"]["egress"]:
|
||||
if r.get("toEndpoints"):
|
||||
r["toEndpoints"][0]["matchLabels"]["io.kubernetes.pod.namespace"] = "default"
|
||||
for name, fn in (("world entity added", m_eg_world), ("LAN not excluded", m_eg_lan),
|
||||
("port 80 added", m_eg_port), ("endpoints moved", m_eg_endpoint)):
|
||||
m = copy.deepcopy(eg)
|
||||
fn(m)
|
||||
check(bool(eval_egress(m)), f"(5) egress evaluator missed mutant: {name}")
|
||||
|
||||
if CHECKS < 18:
|
||||
FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -0,0 +1,34 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the CVE scanner's supply chain and privileges are what docs/17 "Security of the scanner
|
||||
# itself" says (R-VULN-09). Offline, no cluster.
|
||||
#
|
||||
# WHY SCOPE, NOT REQUIREMENTS: the property spans platform/vuln-scanner and renovate.json5. A
|
||||
# requirement-scoped selection would miss a Renovate edit that quietly lets the scanner auto-merge.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-supply-chain
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-09: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-09"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
The scanner image is pinned by digest and is not a banned Trivy version; Renovate never
|
||||
auto-merges it and proposes it only after 7 days; its RBAC holds get/list/watch only and no
|
||||
Secret; the pod is unprivileged with bounded resources and a size-limited cache. Each is
|
||||
proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-supply-chain-and-privileges
|
||||
description: >-
|
||||
Parse platform/vuln-scanner/deployment.yaml, rbac.yaml and renovate.json5, assert the
|
||||
digest/version/Renovate/RBAC/security-context contract, and run each evaluator on mutated
|
||||
copies to prove it can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-scanner-supply-chain
|
||||
# Serves: R-VULN-09 (docs/17 "Security of the scanner itself").
|
||||
#
|
||||
# GIVEN the repository as authored,
|
||||
# THEN (1) the Trivy image is `ghcr.io/aquasecurity/trivy:<ver>@sha256:<64hex>` — by digest, and a
|
||||
# tag so Renovate has something to follow;
|
||||
# (2) that version is not 0.69.4/0.69.5/0.69.6 (the compromised March 2026 releases);
|
||||
# (3) every pod image is digest-pinned except the oauth2-proxy sidecar, which follows the
|
||||
# per-app sidecar pattern so one Renovate branch moves them all;
|
||||
# (4) renovate.json5 keeps Trivy in the blast-radius list, automerge:false, minimumReleaseAge
|
||||
# of at least 7 days, and no later rule re-enables automerge for it;
|
||||
# (5) rbac.yaml: only ClusterRole/ClusterRoleBinding to the scanner SA, only get/list/watch, no
|
||||
# Secret, no wildcard, and only the resources the inventory needs;
|
||||
# (6) the pod: non-root, read-only rootfs, drop ALL, seccomp RuntimeDefault, no host
|
||||
# namespaces or hostPath, CPU and memory limits on every container, an ephemeral-storage
|
||||
# limit on the scanner, and a size-limited /cache emptyDir.
|
||||
# (8) a renovate.json5 rule covers every file under platform/vuln-scanner: automerge:false,
|
||||
# minimumReleaseAge of at least 7 days, and no later rule turns auto-merge back on (the
|
||||
# python image holds the cluster-read token);
|
||||
# (9) the scanner names the private registry it cannot reach, so that registry reports
|
||||
# credential-required rather than a permanent failed.
|
||||
# AND (7) each evaluator is run on a mutated copy and must flag it (non-vacuity).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy, json, re, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-scanner-supply-chain"
|
||||
FAILURES, CHECKS = [], 0
|
||||
COMP = ROOT / "platform" / "vuln-scanner"
|
||||
BANNED = {"0.69.4", "0.69.5", "0.69.6"}
|
||||
DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
ALLOWED_VERBS = {"get", "list", "watch"}
|
||||
ALLOWED_RESOURCES = {
|
||||
"": {"pods", "nodes"},
|
||||
"apps": {"replicasets", "deployments", "statefulsets", "daemonsets"},
|
||||
"batch": {"jobs", "cronjobs"},
|
||||
"kubevirt.io": {"virtualmachineinstances"},
|
||||
}
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
def docs(path):
|
||||
return [d for d in yaml.safe_load_all(path.read_text()) if isinstance(d, dict)]
|
||||
|
||||
|
||||
for f in ("deployment.yaml", "rbac.yaml"):
|
||||
if not (COMP / f).is_file():
|
||||
print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/{f} is absent", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def split_ref(image):
|
||||
# A digest suffix, then a tag after the last ':' (only when it holds no '/', so a registry port
|
||||
# stays in the repository part).
|
||||
ref, _, digest = image.partition("@")
|
||||
head, colon, tail = ref.rpartition(":")
|
||||
if colon and "/" not in tail:
|
||||
return head, tail, digest or None
|
||||
return ref, None, digest or None
|
||||
|
||||
|
||||
def containers(pod):
|
||||
return (pod.get("initContainers") or []) + (pod.get("containers") or [])
|
||||
|
||||
|
||||
def eval_images(pod):
|
||||
bad = []
|
||||
trivy = [c for c in containers(pod) if split_ref(c["image"])[0] == "ghcr.io/aquasecurity/trivy"]
|
||||
if len(trivy) != 1:
|
||||
bad.append(f"expected one Trivy image, found {len(trivy)}")
|
||||
for c in containers(pod):
|
||||
repo, tag, digest = split_ref(c["image"])
|
||||
if repo == "quay.io/oauth2-proxy/oauth2-proxy":
|
||||
if not tag or tag == "latest":
|
||||
bad.append("oauth2-proxy sidecar has no version tag")
|
||||
continue
|
||||
if not digest or not DIGEST.match(digest):
|
||||
bad.append(f"{c['name']} image {c['image']} is not pinned by digest")
|
||||
if not tag:
|
||||
bad.append(f"{c['name']} image {c['image']} has no tag for Renovate to follow")
|
||||
if repo == "ghcr.io/aquasecurity/trivy" and tag in BANNED:
|
||||
bad.append(f"Trivy {tag} is a banned (compromised) version")
|
||||
return bad
|
||||
|
||||
|
||||
def strip_json5(text):
|
||||
# Drop // comments outside strings, then trailing commas.
|
||||
out, i, in_str, esc = [], 0, False, False
|
||||
while i < len(text):
|
||||
ch = text[i]
|
||||
if in_str:
|
||||
out.append(ch)
|
||||
if esc:
|
||||
esc = False
|
||||
elif ch == "\\":
|
||||
esc = True
|
||||
elif ch == '"':
|
||||
in_str = False
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"':
|
||||
in_str = True
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if text.startswith("//", i):
|
||||
while i < len(text) and text[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
||||
|
||||
|
||||
TRIVY_PKG = "ghcr.io/aquasecurity/trivy"
|
||||
VERSION_UPDATE_RULE_RE = re.compile(r"^(\d+)\s*(day|days)$")
|
||||
|
||||
|
||||
def eval_renovate(rules):
|
||||
bad = []
|
||||
trivy_rules = [r for r in rules if TRIVY_PKG in (r.get("matchPackageNames") or [])]
|
||||
if not trivy_rules:
|
||||
bad.append("no packageRule matches ghcr.io/aquasecurity/trivy (blast-radius list)")
|
||||
return bad
|
||||
if not any(r.get("automerge") is False and "blast-radius" in (r.get("addLabels") or [])
|
||||
for r in trivy_rules):
|
||||
bad.append("Trivy is not in a blast-radius rule with automerge:false")
|
||||
ages = [r["minimumReleaseAge"] for r in trivy_rules if "minimumReleaseAge" in r]
|
||||
if not ages or not any(VERSION_UPDATE_RULE_RE.match(a) and int(VERSION_UPDATE_RULE_RE.match(a).group(1)) >= 7
|
||||
for a in ages):
|
||||
bad.append(f"Trivy has no minimumReleaseAge of at least 7 days (found {ages})")
|
||||
# Renovate applies packageRules in order, later wins; the last rule that names automerge for
|
||||
# Trivy (or matches it generically by update type) must be false.
|
||||
last = None
|
||||
for r in rules:
|
||||
if "automerge" in r and (TRIVY_PKG in (r.get("matchPackageNames") or [])
|
||||
or set(r) <= {"description", "matchUpdateTypes", "automerge", "automergeType", "platformAutomerge"}):
|
||||
last = r
|
||||
if last is None or last.get("automerge") is not False:
|
||||
bad.append("the last packageRule deciding automerge for Trivy leaves it true")
|
||||
return bad
|
||||
|
||||
|
||||
SCANNER_FILES = "platform/vuln-scanner/**"
|
||||
|
||||
|
||||
def eval_renovate_files(rules):
|
||||
positions = [i for i, r in enumerate(rules) if SCANNER_FILES in (r.get("matchFileNames") or [])]
|
||||
if not positions:
|
||||
return [f"no packageRule has matchFileNames {SCANNER_FILES}"]
|
||||
bad = []
|
||||
rule = rules[positions[-1]]
|
||||
if rule.get("automerge") is not False:
|
||||
bad.append("the platform/vuln-scanner file rule is not automerge:false")
|
||||
age = VERSION_UPDATE_RULE_RE.match(rule.get("minimumReleaseAge") or "")
|
||||
if not age or int(age.group(1)) < 7:
|
||||
bad.append("the platform/vuln-scanner file rule has no minimumReleaseAge of at least 7 days")
|
||||
if any(r.get("automerge") is True for r in rules[positions[-1] + 1:]):
|
||||
bad.append("a rule after the platform/vuln-scanner file rule re-enables auto-merge")
|
||||
return bad
|
||||
|
||||
|
||||
def eval_private_registries(pod):
|
||||
scanner = next(c for c in pod["containers"] if c["name"] == "scanner")
|
||||
env = {e["name"]: e.get("value") for e in scanner.get("env") or []}
|
||||
if env.get("VULN_PRIVATE_REGISTRIES") != "gitea.${BASE_DOMAIN}":
|
||||
return ["the scanner does not name the private registry gitea.${BASE_DOMAIN}"]
|
||||
return []
|
||||
|
||||
|
||||
def eval_rbac(rbac):
|
||||
bad = []
|
||||
croles = [d for d in rbac if d.get("kind") == "ClusterRole"]
|
||||
if len(croles) != 1:
|
||||
bad.append(f"expected one ClusterRole, found {len(croles)}")
|
||||
return bad
|
||||
for rule in croles[0].get("rules") or []:
|
||||
verbs = set(rule.get("verbs") or [])
|
||||
if verbs - ALLOWED_VERBS:
|
||||
bad.append(f"rule grants non-read verbs {sorted(verbs - ALLOWED_VERBS)}")
|
||||
if "*" in verbs:
|
||||
bad.append("rule grants the verb wildcard")
|
||||
for group in rule.get("apiGroups") or []:
|
||||
if group == "*" or group not in ALLOWED_RESOURCES:
|
||||
bad.append(f"rule names apiGroup {group!r}")
|
||||
continue
|
||||
for res in rule.get("resources") or []:
|
||||
if res == "*" or res not in ALLOWED_RESOURCES[group]:
|
||||
bad.append(f"rule grants resource {group}/{res}")
|
||||
if "secrets" in (rule.get("resources") or []):
|
||||
bad.append("rule grants Secret access")
|
||||
kinds = {d.get("kind") for d in rbac}
|
||||
if kinds - {"ServiceAccount", "ClusterRole", "ClusterRoleBinding"}:
|
||||
bad.append(f"unexpected RBAC kinds {sorted(kinds)}")
|
||||
for b in (d for d in rbac if d.get("kind") == "ClusterRoleBinding"):
|
||||
if (b.get("roleRef") or {}).get("name") != croles[0]["metadata"]["name"]:
|
||||
bad.append("the ClusterRoleBinding does not bind the scanner's ClusterRole")
|
||||
return bad
|
||||
|
||||
|
||||
def eval_pod(pod):
|
||||
bad = []
|
||||
psc = pod.get("securityContext") or {}
|
||||
if psc.get("runAsNonRoot") is not True:
|
||||
bad.append("pod runAsNonRoot is not true")
|
||||
if (psc.get("seccompProfile") or {}).get("type") != "RuntimeDefault":
|
||||
bad.append("pod seccompProfile is not RuntimeDefault")
|
||||
for f in ("hostNetwork", "hostPID", "hostIPC"):
|
||||
if pod.get(f):
|
||||
bad.append(f"pod sets {f}")
|
||||
if any("hostPath" in v for v in pod.get("volumes") or []):
|
||||
bad.append("pod mounts a hostPath volume")
|
||||
for c in containers(pod):
|
||||
sc = c.get("securityContext") or {}
|
||||
if sc.get("readOnlyRootFilesystem") is not True:
|
||||
bad.append(f"{c['name']} root filesystem is writable")
|
||||
if sc.get("allowPrivilegeEscalation") is not False:
|
||||
bad.append(f"{c['name']} allows privilege escalation")
|
||||
if "ALL" not in ((sc.get("capabilities") or {}).get("drop") or []):
|
||||
bad.append(f"{c['name']} does not drop ALL capabilities")
|
||||
limits = (c.get("resources") or {}).get("limits") or {}
|
||||
for lim in ("cpu", "memory"):
|
||||
if lim not in limits:
|
||||
bad.append(f"{c['name']} has no {lim} limit")
|
||||
if c["name"] == "scanner" and "ephemeral-storage" not in limits:
|
||||
bad.append("the scanner has no ephemeral-storage limit")
|
||||
vols = {v["name"]: v for v in pod.get("volumes") or []}
|
||||
if not (vols.get("cache", {}).get("emptyDir") or {}).get("sizeLimit"):
|
||||
bad.append("the /cache emptyDir has no sizeLimit (Longhorn or etcd must never hold it)")
|
||||
return bad
|
||||
|
||||
|
||||
dep = next(d for d in docs(COMP / "deployment.yaml") if d.get("kind") == "Deployment")
|
||||
pod = dep["spec"]["template"]["spec"]
|
||||
for v in eval_images(pod) + eval_pod(pod):
|
||||
check(False, f"(1-3,6) {v}")
|
||||
check(dep["spec"].get("replicas") == 1, "(6) one replica (one RWO report store)")
|
||||
check((dep["spec"].get("strategy") or {}).get("type") == "Recreate", "(6) Recreate strategy")
|
||||
|
||||
renovate = json.loads(strip_json5((ROOT / "renovate.json5").read_text()))
|
||||
rules = renovate.get("packageRules") or []
|
||||
check(len(rules) > 5, "(4) parsed renovate.json5 packageRules (parser degenerated)")
|
||||
for v in eval_renovate(rules):
|
||||
check(False, f"(4) {v}")
|
||||
for v in eval_renovate_files(rules):
|
||||
check(False, f"(8) {v}")
|
||||
for v in eval_private_registries(pod):
|
||||
check(False, f"(9) {v}")
|
||||
|
||||
rbac = docs(COMP / "rbac.yaml")
|
||||
for v in eval_rbac(rbac):
|
||||
check(False, f"(5) {v}")
|
||||
|
||||
# (7) non-vacuity
|
||||
def mutated(fn, base):
|
||||
m = copy.deepcopy(base)
|
||||
fn(m)
|
||||
return m
|
||||
|
||||
# Image mutants
|
||||
def img_digest(p): p["initContainers"][0]["image"] = "ghcr.io/aquasecurity/trivy:0.76.1"
|
||||
def img_banned(p): p["initContainers"][0]["image"] = re.sub(r":[0-9.]+@", ":0.69.5@", p["initContainers"][0]["image"])
|
||||
def img_nopin(p): p["containers"][0]["image"] = "docker.io/library/python:3.14.9-alpine"
|
||||
for name, fn in (("digest stripped from Trivy", img_digest), ("banned Trivy version", img_banned),
|
||||
("python image unpinned", img_nopin)):
|
||||
check(bool(eval_images(mutated(fn, pod))), f"(7) image evaluator missed mutant: {name}")
|
||||
|
||||
# RBAC mutants
|
||||
def rbac_secret(r):
|
||||
role = next(d for d in r if d.get("kind") == "ClusterRole")
|
||||
role["rules"][0]["resources"].append("secrets")
|
||||
def rbac_write(r):
|
||||
role = next(d for d in r if d.get("kind") == "ClusterRole")
|
||||
role["rules"][0]["verbs"].append("delete")
|
||||
def rbac_wild(r):
|
||||
role = next(d for d in r if d.get("kind") == "ClusterRole")
|
||||
role["rules"][1]["resources"] = ["*"]
|
||||
for name, fn in (("Secret resource granted", rbac_secret), ("write verb granted", rbac_write),
|
||||
("resource wildcard granted", rbac_wild)):
|
||||
check(bool(eval_rbac(mutated(fn, rbac))), f"(7) RBAC evaluator missed mutant: {name}")
|
||||
|
||||
# Renovate mutants
|
||||
def ren_auto(r):
|
||||
for x in r:
|
||||
if TRIVY_PKG in (x.get("matchPackageNames") or []):
|
||||
x["automerge"] = True
|
||||
def ren_age(r):
|
||||
for x in r:
|
||||
if TRIVY_PKG in (x.get("matchPackageNames") or []) and "minimumReleaseAge" in x:
|
||||
x["minimumReleaseAge"] = "2 days"
|
||||
def ren_drop(r):
|
||||
r[:] = [x for x in r if TRIVY_PKG not in (x.get("matchPackageNames") or [])]
|
||||
def ren_later(r):
|
||||
r.append({"matchUpdateTypes": ["digest"], "automerge": True})
|
||||
for name, fn in (("Trivy automerge flipped true", ren_auto), ("minimumReleaseAge shortened", ren_age),
|
||||
("Trivy removed from Renovate rules", ren_drop), ("later auto-merge rule appended", ren_later)):
|
||||
check(bool(eval_renovate(mutated(fn, rules))), f"(7) Renovate evaluator missed mutant: {name}")
|
||||
|
||||
# Renovate file-rule mutants
|
||||
def files_rule(r):
|
||||
return next(x for x in r if SCANNER_FILES in (x.get("matchFileNames") or []))
|
||||
def files_auto(r): files_rule(r)["automerge"] = True
|
||||
def files_age(r): files_rule(r)["minimumReleaseAge"] = "1 day"
|
||||
def files_drop(r): r.remove(files_rule(r))
|
||||
def files_later(r): r.append({"matchDatasources": ["docker"], "automerge": True})
|
||||
for name, fn in (("file rule automerge true", files_auto), ("file rule age shortened", files_age),
|
||||
("file rule removed", files_drop), ("later rule re-enables auto-merge", files_later)):
|
||||
check(bool(eval_renovate_files(mutated(fn, rules))), f"(7) file-rule evaluator missed mutant: {name}")
|
||||
|
||||
def reg_drop(p):
|
||||
next(c for c in p["containers"] if c["name"] == "scanner")["env"][:] = [
|
||||
e for e in next(c for c in p["containers"] if c["name"] == "scanner")["env"]
|
||||
if e["name"] != "VULN_PRIVATE_REGISTRIES"]
|
||||
check(bool(eval_private_registries(mutated(reg_drop, pod))), "(7) private-registry evaluator missed the removal")
|
||||
|
||||
# Pod mutants
|
||||
def pod_root(p): p["securityContext"]["runAsNonRoot"] = False
|
||||
def pod_host(p): p["hostNetwork"] = True
|
||||
def pod_rw(p): p["containers"][0]["securityContext"]["readOnlyRootFilesystem"] = False
|
||||
def pod_nolim(p): del p["containers"][0]["resources"]["limits"]["memory"]
|
||||
def pod_nocap(p):
|
||||
for v in p["volumes"]:
|
||||
if v["name"] == "cache":
|
||||
v["emptyDir"].pop("sizeLimit", None)
|
||||
for name, fn in (("runAsNonRoot off", pod_root), ("hostNetwork on", pod_host), ("rw rootfs", pod_rw),
|
||||
("no memory limit", pod_nolim), ("no cache sizeLimit", pod_nocap)):
|
||||
check(bool(eval_pod(mutated(fn, pod))), f"(7) pod evaluator missed mutant: {name}")
|
||||
|
||||
if CHECKS < 15:
|
||||
FAILURES.append(f"only {CHECKS} assertions ran — the check degenerated")
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -0,0 +1,31 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the CVE triage bot workflow fails loudly and is shaped like Renovate's (R-VULN-05,
|
||||
# R-VULN-07, docs/17 "The triage bot"). Offline, no cluster. Depends on .gitea/workflows/cve-triage.yaml
|
||||
# (the BOT's file); fails until that file is integrated.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-triage-bot-workflow-guards
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-05: "true"
|
||||
conformance.home.k3s/R-VULN-07: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-05, R-VULN-07"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
The bot workflow runs on the 07:15 cron, has a workflow_dispatch dry-run input, never runs
|
||||
concurrently with itself, and a step fails the job hard when RENOVATE_TOKEN is empty.
|
||||
Proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-workflow-guards
|
||||
description: >-
|
||||
Parse .gitea/workflows/cve-triage.yaml and assert the trigger, concurrency and hard
|
||||
credential failure, then run the evaluator on mutated copies to prove it can fail.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-triage-bot-workflow-guards
|
||||
# Serves: R-VULN-05 (runs on a schedule, fails loudly), R-VULN-07 (bot conventions, docs/17).
|
||||
#
|
||||
# GIVEN .gitea/workflows/cve-triage.yaml,
|
||||
# THEN (1) it runs on `15 7 * * *` and has a workflow_dispatch `dry_run` input;
|
||||
# (2) concurrency is set (the workflow never runs against itself);
|
||||
# (3) a step fails the job HARD when RENOVATE_TOKEN is empty (exit non-zero, not skip);
|
||||
# AND (4) the evaluator is run on mutated copies and must flag each (non-vacuity).
|
||||
# A missing workflow file is a FAIL: this is whole-repo scoped and has no skip path.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy, re, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-triage-bot-workflow-guards"
|
||||
FAILURES, CHECKS = [], 0
|
||||
WF = ROOT / ".gitea" / "workflows" / "cve-triage.yaml"
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
if not WF.is_file():
|
||||
print(f"CHECK FAIL ({NAME}): .gitea/workflows/cve-triage.yaml is absent (the BOT's deliverable); "
|
||||
f"whether it fails on a missing credential cannot be asserted without it", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
doc = yaml.safe_load(WF.read_text())
|
||||
|
||||
|
||||
def evaluate(doc):
|
||||
bad = []
|
||||
triggers = doc.get(True) if True in doc else doc.get("on") # YAML reads `on:` as boolean True
|
||||
if not isinstance(triggers, dict):
|
||||
return ["no triggers parsed"]
|
||||
if "15 7 * * *" not in [s.get("cron") for s in triggers.get("schedule") or [] if isinstance(s, dict)]:
|
||||
bad.append("no schedule with cron 15 7 * * *")
|
||||
inputs = ((triggers.get("workflow_dispatch") or {}).get("inputs") or {})
|
||||
if "dry_run" not in inputs:
|
||||
bad.append("workflow_dispatch has no dry_run input")
|
||||
conc = doc.get("concurrency")
|
||||
jobs = doc.get("jobs") or {}
|
||||
if not conc and not any(isinstance(j, dict) and j.get("concurrency") for j in jobs.values()):
|
||||
bad.append("no concurrency set (the bot must not run against itself)")
|
||||
hard_fail = False
|
||||
for j in jobs.values():
|
||||
for step in (j.get("steps") or []) if isinstance(j, dict) else []:
|
||||
run = str(step.get("run", ""))
|
||||
if ("RENOVATE_TOKEN" in run and re.search(r"(-z|!\s*-n)[^\n]*RENOVATE_TOKEN|RENOVATE_TOKEN[^\n]*(-z)", run)
|
||||
and re.search(r"exit\s+[1-9]", run)):
|
||||
hard_fail = True
|
||||
if not hard_fail:
|
||||
bad.append("no step fails hard (exit non-zero) when RENOVATE_TOKEN is empty")
|
||||
return bad
|
||||
|
||||
|
||||
for v in evaluate(doc):
|
||||
check(False, f"(1-3) {v}")
|
||||
check(isinstance(doc.get("jobs"), dict) and doc["jobs"], "(1) the workflow declares jobs")
|
||||
|
||||
# (4) non-vacuity
|
||||
def mutate(fn):
|
||||
m = copy.deepcopy(doc)
|
||||
fn(m)
|
||||
return m
|
||||
|
||||
def trig(m):
|
||||
return m.get(True) if True in m else m.get("on")
|
||||
|
||||
def no_cron(m): trig(m).pop("schedule", None)
|
||||
def no_dry(m): trig(m)["workflow_dispatch"] = {}
|
||||
def no_conc(m):
|
||||
m.pop("concurrency", None)
|
||||
for j in m.get("jobs", {}).values():
|
||||
if isinstance(j, dict):
|
||||
j.pop("concurrency", None)
|
||||
def no_guard(m):
|
||||
for j in m.get("jobs", {}).values():
|
||||
for step in j.get("steps") or []:
|
||||
if "RENOVATE_TOKEN" in str(step.get("run", "")):
|
||||
step["run"] = 'echo "::warning::RENOVATE_TOKEN unset, skipping"'
|
||||
for name, fn in (("cron removed", no_cron), ("dry_run input removed", no_dry),
|
||||
("concurrency removed", no_conc), ("credential check softened", no_guard)):
|
||||
check(bool(evaluate(mutate(fn))), f"(4) evaluator missed mutant: {name}")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
@@ -62,7 +62,7 @@ spec:
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
+1
-1
@@ -85,7 +85,7 @@ spec:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -85,7 +85,7 @@ spec:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --http-address=0.0.0.0:4180
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-acceptances
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-08: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-08"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given acceptance files, When parsed and applied, Then expiry reverts to open and malformed entries never hide a finding.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-08: acceptance parsing, expiry and fail-safe behaviour."""
|
||||
import sys
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import acceptances # noqa: E402
|
||||
import report # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
TODAY = date(2026, 11, 9)
|
||||
NOW = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
def parse(text):
|
||||
return acceptances.parse_text(text, TODAY)
|
||||
|
||||
|
||||
entries, errors = parse("""# header comment
|
||||
- id: CVE-2026-1
|
||||
image: docker.io/library/nginx # exact match
|
||||
reason: not reachable - feature disabled
|
||||
expires: 2027-02-01
|
||||
|
||||
- id: CVE-2026-2
|
||||
reason: "platform wide"
|
||||
expires: 2026-12-01
|
||||
""")
|
||||
check("valid flat entries parse with no errors", len(entries) == 2 and errors == [])
|
||||
check("inline comment is stripped", entries[0]["image"] == "docker.io/library/nginx")
|
||||
check("omitted image means platform-wide", entries[1]["image"] is None)
|
||||
check("empty list form is accepted", parse("# none\n[]\n") == ([], []))
|
||||
check("empty file is accepted", parse("") == ([], []))
|
||||
|
||||
for name, text in (
|
||||
("missing reason", "- id: CVE-1\n expires: 2026-12-01\n"),
|
||||
("missing expires", "- id: CVE-1\n reason: x\n"),
|
||||
("expires beyond 180 days", "- id: CVE-1\n reason: x\n expires: 2027-06-01\n"),
|
||||
("unknown key", "- id: CVE-1\n reason: x\n expires: 2026-12-01\n scope: all\n"),
|
||||
("not a date", "- id: CVE-1\n reason: x\n expires: soon\n"),
|
||||
("impossible date", "- id: CVE-1\n reason: x\n expires: 2026-13-45\n"),
|
||||
("duplicate key", "- id: CVE-1\n reason: x\n reason: y\n expires: 2026-12-01\n"),
|
||||
("flow value", "- id: CVE-1\n reason: [x]\n expires: 2026-12-01\n"),
|
||||
):
|
||||
found, problems = parse(text)
|
||||
check("rejected: " + name, found == [] and len(problems) == 1)
|
||||
|
||||
found, problems = parse("- id: CVE-1\n reason: x\n expires: 2026-12-01\n- id: bad entry\n expires: 2026-12-01\n")
|
||||
check("one bad entry is ignored, the valid one kept", len(found) == 1 and len(problems) == 1)
|
||||
found, problems = parse("- id: CVE-1\n reason: x\n")
|
||||
check("bad indentation invalidates the whole file", found == [] and problems)
|
||||
found, problems = parse("garbage\n- id: CVE-1\n reason: x\n expires: 2026-12-01\n")
|
||||
check("a stray line invalidates the whole file", found == [] and problems)
|
||||
check("missing file is an error, not an empty allowlist", acceptances.load("/nonexistent/acceptances.yaml")[1])
|
||||
|
||||
entry = {"id": "CVE-1", "image": None, "reason": "r", "expires": date(2026, 11, 9)}
|
||||
check("acceptance holds on its expiry day", acceptances.match([entry], "CVE-1", "any/image", TODAY) is entry)
|
||||
check("acceptance gone the day after", acceptances.match([entry], "CVE-1", "any/image", TODAY + timedelta(days=1)) is None)
|
||||
scoped = {"id": "CVE-1", "image": "docker.io/library/nginx", "reason": "r", "expires": date(2026, 12, 1)}
|
||||
check("image match is exact", acceptances.match([scoped], "CVE-1", "docker.io/library/nginx-extra", TODAY) is None)
|
||||
check("image match accepts the exact image", acceptances.match([scoped], "CVE-1", "docker.io/library/nginx", TODAY) is scoped)
|
||||
|
||||
|
||||
def state_with(findings):
|
||||
target = {"id": "docker.io/library/nginx@" + A1, "type": "image", "image": "docker.io/library/nginx", "digest": A1,
|
||||
"ref": "", "reason": None, "workloads": []}
|
||||
return {
|
||||
"inventory": [target], "database": {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"},
|
||||
"run": {"startedAt": "2026-11-09T09:00:00Z", "finishedAt": "2026-11-09T09:30:00Z", "status": "complete"},
|
||||
"remediation": [], "inventoryError": None, "loadError": None,
|
||||
"scans": {A1: {"firstDiscoveredAt": "2026-11-01T00:00:00Z", "lastInventoriedAt": "2026-11-09T09:00:00Z",
|
||||
"lastAttemptAt": "2026-11-09T09:00:00Z", "lastSuccessAt": "2026-11-09T09:00:00Z", "outcome": "ok",
|
||||
"reason": None, "failedSince": None, "findings": findings,
|
||||
"firstSeen": {f["id"]: "2026-11-01T00:00:00Z" for f in findings}}},
|
||||
}
|
||||
|
||||
|
||||
FINDING = {"id": "CVE-1", "severity": "CRITICAL", "cvss": 9.0, "package": "p", "installedVersion": "1", "fixedVersion": "2",
|
||||
"title": "t", "url": ""}
|
||||
ST = state_with([FINDING])
|
||||
|
||||
|
||||
def status(when, accepted, errs=()):
|
||||
built = report.build(ST, accepted, list(errs), when)
|
||||
return built, built["findings"][0]
|
||||
|
||||
|
||||
def reason_entry(length):
|
||||
return acceptances.parse_text("- id: CVE-1\n reason: " + "r" * length + "\n expires: 2026-12-01\n", NOW.date())
|
||||
|
||||
|
||||
found, problems = reason_entry(3000)
|
||||
check("a reason over 2048 characters ignores the entry and reports it", found == [] and any("2048" in p for p in problems))
|
||||
found, problems = reason_entry(2048)
|
||||
check("mutation: a reason of exactly 2048 characters is valid", len(found) == 1 and problems == [])
|
||||
|
||||
entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)}
|
||||
built, finding = status(NOW, [entry])
|
||||
check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted"
|
||||
and finding["acceptance"] == {"reason": "not reachable", "expires": "2026-11-09"})
|
||||
check("accepted report is schema-valid", report.validate(built) == [])
|
||||
_, finding = status(NOW + timedelta(days=1), [entry])
|
||||
check("expired acceptance reverts to the finding's own status", finding["status"] == "fix-available" and "acceptance" not in finding)
|
||||
_, finding = status(NOW, [])
|
||||
check("mutation: no acceptance leaves it open/fix-available", finding["status"] == "fix-available")
|
||||
built, finding = status(NOW, [], ["bad entry"])
|
||||
check("an invalid entry degrades the run", built["run"]["status"] == "degraded")
|
||||
check("an invalid entry never hides the finding", finding["status"] == "fix-available")
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,28 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-api-metrics
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-02: "true"
|
||||
conformance.home.k3s/R-VULN-03: "true"
|
||||
conformance.home.k3s/R-VULN-05: "true"
|
||||
conformance.home.k3s/R-VULN-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-02, R-VULN-03, R-VULN-05, R-VULN-06"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given a running engine, When the HTTP surfaces are exercised, Then the API validates, metrics are aggregates only, and listeners answer during a scan.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-02/03/05/06: internal API, metrics, publish refusal, and HTTP during a scan."""
|
||||
import copy
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import main # noqa: E402
|
||||
import metrics # noqa: E402
|
||||
import report # noqa: E402
|
||||
import server # noqa: E402
|
||||
import state as st # noqa: E402
|
||||
|
||||
logging.disable(logging.CRITICAL)
|
||||
FAILED = []
|
||||
NOW = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = NOW
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
|
||||
class Client:
|
||||
def list_all(self, path):
|
||||
if path.endswith("pods"):
|
||||
return [{"metadata": {"namespace": "apps", "name": "p"}, "spec": {"containers": [{"name": "c", "image": "nginx:1"}]},
|
||||
"status": {"containerStatuses": [{"name": "c", "imageID": "docker.io/library/nginx@" + A1}]}}]
|
||||
return []
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Blocks on release while a scan is 'running', so a test can prove the HTTP path stays live."""
|
||||
|
||||
def __init__(self):
|
||||
self.entered = threading.Event()
|
||||
self.release = threading.Event()
|
||||
self.release.set()
|
||||
self.result = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-1", "Severity": "CRITICAL", "PkgName": "openssl", "InstalledVersion": "1",
|
||||
"FixedVersion": "2", "Title": "t", "PrimaryURL": "https://example.test/1"}]}]}
|
||||
|
||||
def download_db(self):
|
||||
pass
|
||||
|
||||
def db_info(self):
|
||||
return {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"}
|
||||
|
||||
def scan(self, _reference):
|
||||
self.entered.set()
|
||||
self.release.wait(10)
|
||||
return self.result
|
||||
|
||||
def clean_cache(self):
|
||||
pass
|
||||
|
||||
|
||||
def request(port, method, path, body=None, headers=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
||||
conn.request(method, path, body, headers or {})
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
conn.close()
|
||||
return resp.status, data, resp
|
||||
|
||||
|
||||
def post(port, document, content_type="application/json"):
|
||||
return request(port, "POST", "/triage", json.dumps(document), {"Content-Type": content_type})
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
accept = os.path.join(tmp, "a.yaml")
|
||||
Path(accept).write_text("[]\n")
|
||||
clock = Clock()
|
||||
runner = Runner()
|
||||
eng = main.Engine(tmp, Client(), runner, accept, clock=clock, publish_seconds=0)
|
||||
servers = server.start_all(eng, ("127.0.0.1", 0), ("127.0.0.1", 0), ("127.0.0.1", 0))
|
||||
page_port, internal, metrics_port = (s.server_address[1] for s in servers)
|
||||
|
||||
status, body, _ = request(internal, "GET", "/report.json")
|
||||
check("report.json is 503 before any report", status == 503)
|
||||
status, body, resp = request(page_port, "GET", "/")
|
||||
check("page is 503 before any report, with the CSP", status == 503 and resp.getheader("Content-Security-Policy"))
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("last-complete-run metric is absent before the first run", "last_complete_run" not in text)
|
||||
check("triage heartbeat metric is absent before the first POST", "triage_heartbeat" not in text)
|
||||
|
||||
# HTTP must stay live while a scan is blocked mid-flight.
|
||||
runner.release.clear()
|
||||
worker = threading.Thread(target=eng.tick, daemon=True)
|
||||
worker.start()
|
||||
check("the scan is genuinely in flight", runner.entered.wait(10))
|
||||
status_h, _, _ = request(internal, "GET", "/healthz")
|
||||
status_p, _, _ = request(page_port, "GET", "/healthz")
|
||||
status_m, _, _ = request(metrics_port, "GET", "/metrics")
|
||||
check("listeners answer during a scan", (status_h, status_p, status_m) == (200, 200, 200))
|
||||
runner.release.set()
|
||||
worker.join(10)
|
||||
check("the tick then completes", not worker.is_alive())
|
||||
|
||||
status, body, _ = request(internal, "GET", "/report.json")
|
||||
check("report.json serves a schema-valid report after a tick", status == 200 and report.validate(json.loads(body)) == [])
|
||||
|
||||
# Page and CSP over HTTP
|
||||
status, body, resp = request(page_port, "GET", "/?view=inventory")
|
||||
check("page served over HTTP carries the CSP and nosniff",
|
||||
status == 200 and resp.getheader("Content-Security-Policy").startswith("default-src 'none'") and resp.getheader("X-Content-Type-Options") == "nosniff")
|
||||
status, body, resp = request(page_port, "GET", "/missing")
|
||||
check("a 404 from the page listener also carries the CSP", status == 404 and resp.getheader("Content-Security-Policy"))
|
||||
|
||||
status, body, resp = request(page_port, "POST", "/")
|
||||
check("an unsupported method on the page listener also carries the CSP", status == 501 and resp.getheader("Content-Security-Policy"))
|
||||
|
||||
# Triage API
|
||||
valid = {"heartbeatAt": "2026-11-09T11:00:00Z", "remediation": [
|
||||
{"image": "docker.io/library/nginx", "state": "pr-open", "url": "https://gitea.test/pulls/9", "updatedAt": "2026-11-09T11:00:00Z"}]}
|
||||
status, body, _ = post(internal, valid)
|
||||
check("valid triage is 204", status == 204 and body == b"")
|
||||
check("remediation surfaces in the report", eng.report["remediation"][0]["state"] == "pr-open")
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("heartbeat metric appears and uses server receipt time, not the bot's", f"triage_heartbeat_timestamp_seconds {int(NOW.timestamp())}" in text)
|
||||
for name, mutate in (
|
||||
("unknown top-level key", lambda d: d.update(extra=1)),
|
||||
("missing heartbeat", lambda d: d.pop("heartbeatAt")),
|
||||
("unknown remediation key", lambda d: d["remediation"][0].update(x=1)),
|
||||
("bad remediation state", lambda d: d["remediation"][0].update(state="merged")),
|
||||
("javascript url", lambda d: d["remediation"][0].update(url="javascript:x")),
|
||||
("bad timestamp", lambda d: d.update(heartbeatAt="yesterday")),
|
||||
):
|
||||
document = copy.deepcopy(valid)
|
||||
mutate(document)
|
||||
status, _, _ = post(internal, document)
|
||||
check("triage rejects " + name, status == 400)
|
||||
for name, url in (("an uppercase scheme", "HTTP://gitea.test/pulls/1"), ("an empty url", ""), ("a javascript url", "javascript:alert(1)")):
|
||||
bad = json.loads(json.dumps(valid))
|
||||
bad["remediation"][0]["url"] = url
|
||||
check("triage rejects " + name, post(internal, bad)[0] == 400)
|
||||
check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415)
|
||||
status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"})
|
||||
check("triage rejects an oversized body", status == 413)
|
||||
many = {"heartbeatAt": "2026-11-09T11:00:00Z", "remediation": [valid["remediation"][0]] * 5001}
|
||||
check("triage rejects more than 5000 entries", post(internal, many)[0] in (400, 413))
|
||||
check("rejected triage leaves state untouched", eng.state["remediation"][0]["url"] == "https://gitea.test/pulls/9")
|
||||
|
||||
# Metrics labels and values
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
allowed = {"state", "severity", "fixable"}
|
||||
labels = {k for line in text.splitlines() for k in re.findall(r"([A-Za-z_]+)=\"", line)}
|
||||
check("metric labels are within the allowlist", labels and labels <= allowed)
|
||||
check("no metric carries a CVE, package, image or workload", not re.search(r"CVE-|openssl|nginx|apps", text))
|
||||
for state in report.STATES:
|
||||
check("targets gauge present for " + state, 'vuln_scanner_targets{state="' + state + '"}' in text)
|
||||
check("critical fix findings counted with fixable true", 'vuln_scanner_findings{severity="CRITICAL",fixable="true"} 1' in text)
|
||||
check("every severity and fixability appears", text.count("vuln_scanner_findings{") == 10)
|
||||
check("failed-over-24h gauge exists", "vuln_scanner_failed_targets_over_24h 0" in text)
|
||||
check("critical fix overdue is 0 inside the 72h grace", "vuln_scanner_critical_fix_overdue 0" in text)
|
||||
clock.now += timedelta(hours=73)
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("critical fix overdue is 1 after 72h", "vuln_scanner_critical_fix_overdue 1" in text)
|
||||
check("last-complete-run metric present after a complete run", "vuln_scanner_last_complete_run_timestamp_seconds" in text)
|
||||
for srv in servers:
|
||||
srv.shutdown()
|
||||
|
||||
# Invalid report: refuse and keep the old one.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
good = {"schemaVersion": 1, "generatedAt": "2026-11-09T10:00:00Z", "database": {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"},
|
||||
"run": {"startedAt": "2026-11-09T09:00:00Z", "finishedAt": "2026-11-09T09:30:00Z", "status": "complete"},
|
||||
"targets": [], "findings": [], "remediation": []}
|
||||
check("a valid report publishes", report.publish(good, tmp) is True)
|
||||
before = Path(tmp, "report.json").read_bytes()
|
||||
for name, mutate in (
|
||||
("wrong schemaVersion", lambda r: r.update(schemaVersion=2)),
|
||||
("extra top-level key", lambda r: r.update(extra=1)),
|
||||
("bad timestamp", lambda r: r.update(generatedAt="soon")),
|
||||
("unknown state", lambda r: r.update(targets=[{"id": "x", "type": "image", "image": "x", "digest": None, "state": "fine",
|
||||
"reason": None, "lastScannedAt": None, "workloads": []}])),
|
||||
("dangling targetId", lambda r: r.update(findings=[{"id": "CVE-1", "targetId": "nope", "severity": "LOW", "cvss": None,
|
||||
"package": "p", "installedVersion": "1", "fixedVersion": None, "title": "t",
|
||||
"url": "", "firstSeenAt": "2026-11-01T00:00:00Z", "status": "open"}])),
|
||||
):
|
||||
broken = copy.deepcopy(good)
|
||||
mutate(broken)
|
||||
check("invalid report refused: " + name, report.publish(broken, tmp) is False)
|
||||
check("an invalid report writes nothing", Path(tmp, "report.json").read_bytes() == before)
|
||||
check("accepted without acceptance is invalid", report.validate({**good, "targets": [{"id": "x", "type": "image", "image": "x", "digest": None,
|
||||
"state": "scanned", "reason": None, "lastScannedAt": None, "workloads": []}], "findings": [{"id": "CVE-1", "targetId": "x", "severity": "LOW",
|
||||
"cvss": None, "package": "p", "installedVersion": "1", "fixedVersion": None, "title": "t", "url": "", "firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"status": "accepted"}]}) != [])
|
||||
Path(tmp, "report.json").write_text("{not json")
|
||||
check("a corrupt published report is not served", report.load_published(tmp) is None)
|
||||
|
||||
# Crash-consistency of state: the temp+rename leaves no temp files behind and the file parses.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
store = st.Store(tmp)
|
||||
state = st.new_state()
|
||||
state["triageHeartbeatAt"] = "2026-11-09T11:00:00Z"
|
||||
store.save(state)
|
||||
check("state saves with no temp file left behind", sorted(os.listdir(tmp)) == ["state.json"])
|
||||
check("state survives a reload", st.Store(tmp).load()["triageHeartbeatAt"] == "2026-11-09T11:00:00Z")
|
||||
Path(tmp, "state.json").write_text("not json")
|
||||
loaded = st.Store(tmp).load()
|
||||
check("corrupt state starts empty and marks degraded", loaded["loadError"] and loaded["scans"] == {})
|
||||
check("corrupt state is preserved for inspection", Path(tmp, "state.json.corrupt").exists())
|
||||
|
||||
# Metrics module directly, without the report.
|
||||
empty_state = st.new_state()
|
||||
text = metrics.render(None, empty_state, NOW)
|
||||
check("with no report only the state-derived gauge exists", text == "vuln_scanner_failed_targets_over_24h 0\n")
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,25 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-inventory
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-01"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given recorded API lists, When the inventory is built, Then owner chains, digests, by-tag CronJobs and not-covered targets resolve per R-VULN-01.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-01: inventory from recorded API lists."""
|
||||
import copy
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import inventory # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
CLUSTER = json.loads((HERE / "fixtures/cluster.json").read_text())
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
B2 = "sha256:" + "b2" * 32
|
||||
C3 = "sha256:" + "c3" * 32
|
||||
D4 = "sha256:" + "d4" * 32
|
||||
E5 = "sha256:" + "e5" * 32
|
||||
F6 = "sha256:" + "f6" * 32
|
||||
|
||||
|
||||
def by_id(targets):
|
||||
return {t["id"]: t for t in targets}
|
||||
|
||||
|
||||
def workloads(target):
|
||||
return {(w["namespace"], w["kind"], w["name"]): w["containers"] for w in target["workloads"]}
|
||||
|
||||
|
||||
targets = inventory.build(CLUSTER)
|
||||
ids = by_id(targets)
|
||||
|
||||
nginx = ids["docker.io/library/nginx@" + A1]
|
||||
check("ReplicaSet resolves to Deployment, not the ReplicaSet", workloads(nginx) == {
|
||||
("apps", "Deployment", "web"): ["app"], ("apps", "Deployment", "api"): ["server"]})
|
||||
check("docker-pullable:// prefix stripped and digest taken from imageID", nginx["digest"] == A1)
|
||||
check("init container inventoried", ("apps", "Deployment", "web") in workloads(ids["ghcr.io/org/tool@" + B2]))
|
||||
check("ephemeral container inventoried", "debug" in workloads(ids["docker.io/library/busybox@" + C3])[("apps", "Deployment", "web")])
|
||||
|
||||
backup = ids["ghcr.io/org/backup@" + E5]
|
||||
check("Job resolves to CronJob", workloads(backup) == {("ops", "CronJob", "backup"): ["backup"]})
|
||||
check("a CronJob with a running pod is not also inventoried from its template",
|
||||
not any(t["id"].startswith("ghcr.io/org/backup:") for t in targets))
|
||||
|
||||
nightly = ids["ghcr.io/org/nightly:2.1"]
|
||||
check("idle CronJob inventoried from template with no digest", nightly["digest"] is None
|
||||
and workloads(nightly) == {("ops", "CronJob", "nightly"): ["job"]})
|
||||
|
||||
solo = ids["docker.io/library/alpine:3.20"]
|
||||
check("empty imageID gives a by-tag target from the spec image", solo["digest"] is None)
|
||||
check("bare Pod is its own workload", ("ops", "Pod", "solo") in workloads(solo))
|
||||
check("Succeeded pod's image is not inventoried", not any(F6 in t["id"] for t in targets))
|
||||
check("virt-launcher stays owned by its VirtualMachineInstance",
|
||||
("kubevirt-vms", "VirtualMachineInstance", "vm1") in workloads(ids["quay.io/kubevirt/virt-launcher@sha256:" + "ab" * 32]))
|
||||
vm, node = ids["vm/kubevirt-vms/vm1"], ids["node/prod-1"]
|
||||
check("VM inventoried as non-container target", vm["type"] == "vm" and vm["digest"] is None and vm["reason"])
|
||||
check("node inventoried as non-container target", node["type"] == "node" and node["reason"])
|
||||
|
||||
# Non-vacuous: mutating the fixture changes the answer the assertions above rely on.
|
||||
broken = copy.deepcopy(CLUSTER)
|
||||
broken["replicasets"] = []
|
||||
broken["jobs"] = []
|
||||
broken_ids = by_id(inventory.build(broken))
|
||||
check("mutation: losing the ReplicaSet record leaves the ReplicaSet as the workload",
|
||||
("apps", "ReplicaSet", "web-5f") in workloads(broken_ids["docker.io/library/nginx@" + A1]))
|
||||
check("mutation: losing the Job record leaves the Job as the workload",
|
||||
("ops", "Job", "backup-123") in workloads(broken_ids["ghcr.io/org/backup@" + E5]))
|
||||
check("mutation: a Running phase on the Succeeded pod does inventory its image", any(
|
||||
F6 in t["id"] for t in inventory.build({**CLUSTER, "pods": [
|
||||
{**p, "status": {**p["status"], "phase": "Running"}} if p["metadata"]["name"] == "done-1" else p
|
||||
for p in CLUSTER["pods"]]})))
|
||||
|
||||
for reference, expected in (
|
||||
("nginx:1.27", ("docker.io/library/nginx", "1.27", None)),
|
||||
("nginx", ("docker.io/library/nginx", None, None)),
|
||||
("bitnami/redis:7", ("docker.io/bitnami/redis", "7", None)),
|
||||
("ghcr.io/org/app:v1", ("ghcr.io/org/app", "v1", None)),
|
||||
("registry.local:5000/x/y:2", ("registry.local:5000/x/y", "2", None)),
|
||||
("index.docker.io/library/nginx:1", ("docker.io/library/nginx", "1", None)),
|
||||
("ghcr.io/org/app:v1@" + A1, ("ghcr.io/org/app", "v1", A1)),
|
||||
):
|
||||
check("normalise " + reference, inventory.normalise_reference(reference) == expected)
|
||||
check("bare config id is not a digest", inventory.digest_from_image_id("sha256:" + "a1" * 32) is None)
|
||||
check("empty imageID has no digest", inventory.digest_from_image_id("") is None)
|
||||
|
||||
|
||||
class FakeListClient:
|
||||
"""list_all replaced; proves collect() tolerates exactly a 404 on the VMI API."""
|
||||
|
||||
def __init__(self, vmi_error):
|
||||
self.vmi_error = vmi_error
|
||||
|
||||
def list_all(self, path):
|
||||
if "kubevirt" in path and self.vmi_error:
|
||||
raise self.vmi_error
|
||||
return []
|
||||
|
||||
|
||||
check("VMI API 404 tolerated as no VMs", inventory.collect(FakeListClient(inventory.NotFound("x")))["vmis"] == [])
|
||||
try:
|
||||
inventory.collect(FakeListClient(inventory.InventoryError("403")))
|
||||
check("a non-404 VMI error is not swallowed", False)
|
||||
except inventory.InventoryError:
|
||||
check("a non-404 VMI error is not swallowed", True)
|
||||
|
||||
|
||||
class PagedAPI(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
if self.headers.get("Authorization") != "Bearer fixture-token":
|
||||
self.send_response(403)
|
||||
self.end_headers()
|
||||
return
|
||||
if self.path.startswith("/apis/kubevirt"):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
if "continue=" in self.path:
|
||||
body = {"items": [{"metadata": {"name": "b"}}], "metadata": {}}
|
||||
else:
|
||||
body = {"items": [{"metadata": {"name": "a"}}], "metadata": {"continue": "page2"}}
|
||||
payload = json.dumps(body).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
token = os.path.join(tmp, "token")
|
||||
Path(token).write_text("fixture-token\n")
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), PagedAPI)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
client = inventory.KubeClient("http://127.0.0.1:" + str(server.server_address[1]), token, os.devnull)
|
||||
items = client.list_all("/api/v1/pods")
|
||||
check("pagination follows the continue token with the bearer token", [i["metadata"]["name"] for i in items] == ["a", "b"])
|
||||
check("collect over real HTTP tolerates VMI 404", inventory.collect(client)["vmis"] == [])
|
||||
server.shutdown()
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,395 @@
|
||||
{
|
||||
"cronjobs": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"jobTemplate": {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/backup:1.0",
|
||||
"name": "job"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "nightly",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"jobTemplate": {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/nightly:2.1",
|
||||
"name": "job"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-123",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "CronJob",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-99",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "CronJob",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"nodes": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "prod-1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "prod-2"
|
||||
}
|
||||
}
|
||||
],
|
||||
"pods": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "web-5f"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"ephemeralContainers": [
|
||||
{
|
||||
"image": "busybox:1.37",
|
||||
"name": "debug"
|
||||
}
|
||||
],
|
||||
"initContainers": [
|
||||
{
|
||||
"image": "ghcr.io/org/tool:v2",
|
||||
"name": "init-db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker-pullable://docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"ephemeralContainerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/busybox@sha256:c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3",
|
||||
"name": "debug"
|
||||
}
|
||||
],
|
||||
"initContainerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/tool@sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2",
|
||||
"name": "init-db"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-2",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "web-5f"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"initContainers": [
|
||||
{
|
||||
"image": "ghcr.io/org/tool:v2",
|
||||
"name": "init-db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"initContainerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/tool@sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2",
|
||||
"name": "init-db"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "api-77"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "server"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "server"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "db-0",
|
||||
"namespace": "data",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "StatefulSet",
|
||||
"name": "db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "postgres:16",
|
||||
"name": "postgres"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/postgres@sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"name": "postgres"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-123-x",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Job",
|
||||
"name": "backup-123"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/backup:1.0",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/backup@sha256:e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5",
|
||||
"name": "backup"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "solo",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "alpine:3.20",
|
||||
"name": "toolbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "",
|
||||
"name": "toolbox"
|
||||
}
|
||||
],
|
||||
"phase": "Pending"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "done-1",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Job",
|
||||
"name": "backup-99"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/old-backup:0.1",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/old-backup@sha256:f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6",
|
||||
"name": "backup"
|
||||
}
|
||||
],
|
||||
"phase": "Succeeded"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "virt-launcher-vm1-abc",
|
||||
"namespace": "kubevirt-vms",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "VirtualMachineInstance",
|
||||
"name": "vm1"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "quay.io/kubevirt/virt-launcher:v1.8",
|
||||
"name": "compute"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "quay.io/kubevirt/virt-launcher@sha256:abababababababababababababababababababababababababababababababab",
|
||||
"name": "compute"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
}
|
||||
],
|
||||
"replicasets": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-5f",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "web"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-77",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "api"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"vmis": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "vm1",
|
||||
"namespace": "kubevirt-vms"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-page
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-03: "true"
|
||||
conformance.home.k3s/R-VULN-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-03, R-VULN-04"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given a report fixture, When the page renders, Then all CVEs, containers and workloads appear in all views and hostile input is inert.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
Loaded 100 of 353 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user