Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5.5
|
b855b6b294 |
No files matched your search
@@ -34,13 +34,19 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
# THE TOOLBOX, the image gate.yml already runs, pinned by digest. A job here has no Docker
|
||||
# daemon, so a job container is the only way to choose an image. The toolbox provides
|
||||
# python3, git, curl and jq, pulls anonymously like the gate's, and is normally already in
|
||||
# python3, git, curl and jq, pulls with the gate's credential, and is normally already in
|
||||
# the DinD store. It has no Node, so actions/checkout cannot run; a public python image has
|
||||
# neither Node nor git and would add a second image to keep pinned. The cost is a cold pull
|
||||
# after a runner restart, which the gate pays anyway.
|
||||
container:
|
||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
||||
image: gitea.home.keithjackson.dev/home-network/toolbox@sha256:83a3501efe34fa0b039e3f0d360738686a0f59109866c49779768f5d5ed0ef81
|
||||
# Anonymous registry pulls are refused since REQUIRE_SIGNIN_VIEW (catalog/gitea.yaml); this
|
||||
# job already holds RENOVATE_TOKEN, whose read:package scope renovate.yaml's preflight proves.
|
||||
# See "PULL AUTH" in gate.yml's header.
|
||||
credentials:
|
||||
username: renovate
|
||||
password: ${{ secrets.RENOVATE_TOKEN }}
|
||||
# Without a bound, a wedged run would hold the concurrency group and block every later run.
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
|
||||
@@ -58,11 +58,16 @@
|
||||
# Renovate follows the `main` tag and opens a digest-bump PR, whose OWN gate run then
|
||||
# executes on the new image — the bump tests itself. infra/gitea/registry-retention.sh reads
|
||||
# the same marker, so the pinned digest is never pruned. Never hand-edit it to a tag.
|
||||
# * PULL AUTH: none needed. The Home-Network org is public and the registry issues an
|
||||
# anonymous pull token (measured 2026-11-08: anonymous /v2/token -> manifest 200), so the
|
||||
# runner's DinD pulls it like any public image. If the org is ever made private, add
|
||||
# `credentials:` here with a read:package token secret — the automatic Actions token has no
|
||||
# package scope at /v2/token (see images.yaml's header).
|
||||
# * PULL AUTH: REQUIRED since 2026-11-14. Gitea now runs REQUIRE_SIGNIN_VIEW=true
|
||||
# (catalog/gitea.yaml), whose strict mode refuses the anonymous /v2/token exchange that this
|
||||
# pull used to rely on (measured 2026-11-08: anonymous /v2/token -> manifest 200). The job
|
||||
# therefore declares `credentials:` with RENOVATE_TOKEN — an EXISTING secret whose
|
||||
# read:package scope renovate.yaml's preflight already proves on every run, and the token the
|
||||
# cve-triage job already uses. The automatic Actions token cannot do it: it has no package scope
|
||||
# at /v2/token (see images.yaml's header). act interpolates both fields before the pull and
|
||||
# FAILS THE JOB if either is empty, so a missing secret is a red gate, never a silent
|
||||
# anonymous fallback. A dedicated read:package-only token would be narrower; swapping it in
|
||||
# changes this one line.
|
||||
# * COLD PULL: ~467 MB compressed (1.54 GB unpacked). The DinD store is an emptyDir, so the
|
||||
# first job after a runner-pod restart or a digest bump pays one pull; every other run finds
|
||||
# it cached (forcePull=false), exactly as it already did for the catthehacker image.
|
||||
@@ -111,6 +116,10 @@ jobs:
|
||||
container:
|
||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
||||
image: gitea.home.keithjackson.dev/home-network/toolbox@sha256:83a3501efe34fa0b039e3f0d360738686a0f59109866c49779768f5d5ed0ef81
|
||||
# The registry refuses anonymous pulls (REQUIRE_SIGNIN_VIEW) — see "PULL AUTH" in the header.
|
||||
credentials:
|
||||
username: renovate
|
||||
password: ${{ secrets.RENOVATE_TOKEN }}
|
||||
steps:
|
||||
# Plain git, because actions/checkout is a Node action and the toolbox has no Node. It
|
||||
# reproduces what actions/checkout@v7 with `fetch-depth: 0` did here (read from its logged
|
||||
|
||||
@@ -101,6 +101,8 @@ spec:
|
||||
value: lax
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: 'true'
|
||||
- name: GITEA__service__REQUIRE_SIGNIN_VIEW
|
||||
value: 'true'
|
||||
- name: GITEA_ADMIN_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -131,8 +133,8 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\
|
||||
\ break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n# A postStart hook MUST exit 0 \u2014 a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff). This hook already ends 0 on both paths, but\n# it is made EXPLICIT so the guarantee does not depend on whatever statement happens to\n# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,\n# the moment its OIDC guard was fixed and the block actually ran for the first time.\nexit 0\n"
|
||||
- "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"[email protected]\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- robots.txt: ask polite crawlers to stay away entirely (defence in depth) ---\n# REQUIRE_SIGNIN_VIEW already refuses them, but every refused hit is still a redirect to\n# /user/login that Gitea has to serve. Gitea serves $GITEA_CUSTOM/public/robots.txt\n# (/data/gitea in this image), checked on EVERY request (routers/web/misc/misc.go), and\n# the route is outside the sign-in requirement. Written after the DB wait above, so the\n# entrypoint has already created and chowned /data/gitea. Rewritten each boot, so this\n# declaration is the only source of its content.\nif su-exec git mkdir -p /data/gitea/public \\\n && printf 'User-agent: *\\nDisallow: /\\n' | su-exec git tee /data/gitea/public/robots.txt >/dev/null; then\n echo \"[gitea-bootstrap] robots.txt written (Disallow: /)\"\nelse\n echo \"[gitea-bootstrap] could not write /data/gitea/public/robots.txt\" >&2\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n\
|
||||
# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n# A postStart hook MUST exit 0 \u2014 a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff). This hook already ends 0 on both paths, but\n# it is made EXPLICIT so the guarantee does not depend on whatever statement happens to\n# be last. nextcloud's equivalent hook did depend on that and crashlooped on 2026-09-01,\n# the moment its OIDC guard was fixed and the block actually ran for the first time.\nexit 0\n"
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
|
||||
@@ -135,13 +135,6 @@ spec:
|
||||
# Bump when act-runner-config changes without any other pod-template change, so the
|
||||
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
|
||||
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
|
||||
# Velero's daily schedule is opt-OUT (defaultVolumesToFsBackup: true) and its volume policy
|
||||
# skips only emptyDir and NFS (platform/velero/volume-policies.yaml). dind-storage is now a
|
||||
# Longhorn-backed generic ephemeral volume (see `volumes:`), which that policy does NOT
|
||||
# match, so without this the nightly backup would push the whole Docker image cache to R2
|
||||
# and race this pod's replacement (the PartiallyFailed failure that file documents). It is a
|
||||
# disposable cache: nothing in it is state.
|
||||
backup.velero.io/backup-volumes-excludes: dind-storage
|
||||
spec:
|
||||
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
||||
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
||||
@@ -204,18 +197,12 @@ spec:
|
||||
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
|
||||
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
|
||||
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
|
||||
# ephemeral-storage: /var/lib/docker (images, job containers, their logs) is on the
|
||||
# dind-storage PVC, NOT the node's /var, so this bounds only dockerd's own rootfs
|
||||
# writes and its container log. Before 2026-11-14 the image cache was an unbounded
|
||||
# emptyDir on /var (see `volumes:`).
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 256Mi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
ephemeral-storage: 2Gi
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
@@ -279,132 +266,16 @@ spec:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
ephemeral-storage: 128Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 1Gi
|
||||
# ---- docker-prune: keeps the dind image cache bounded ----------------------------------
|
||||
#
|
||||
# dockerd never garbage-collects images. Every new renovate/toolbox digest (~2 GB each) used
|
||||
# to stack up in /var/lib/docker until the node's /var filled: on 2026-11-12 that tipped
|
||||
# k3s-prod-2 into DiskPressure and the act-runner pod was evicted 63 times. Moving the cache
|
||||
# to its own volume (see `volumes:`) stops it hurting the node; this stops it filling that
|
||||
# volume and failing jobs with ENOSPC instead.
|
||||
#
|
||||
# Every 10 minutes: if the volume is at or above 45% used, `docker system prune --all
|
||||
# --force` removes every image, stopped container, unused network and build cache entry NOT
|
||||
# in use by a container. Images a running job uses are never removed. 45% (not 85%) so the
|
||||
# volume stays above the 50%-free gate of PersistentVolumeFillingUpPredicted
|
||||
# (platform/monitoring/pvc-capacity-alerts.yaml) and a normal refill never pages anyone.
|
||||
# Threshold-triggered, not age-based: `--filter until=` on images is image CREATION time, so
|
||||
# an age-based prune would delete the long-lived job images (catthehacker, the toolbox) between
|
||||
# every pair of jobs and force a re-pull each time.
|
||||
#
|
||||
# Accepted race: a prune landing in the milliseconds between a job's image pull and its
|
||||
# container create removes that image and fails that one job. It needs the volume over the
|
||||
# threshold AT that instant; a re-run passes.
|
||||
#
|
||||
# NO DOLLAR SIGNS, deliberately: clusters/base/apps-wave-1.yaml runs Flux postBuild
|
||||
# substitution over this whole file, comments included, which rewrites shell variables and
|
||||
# command substitutions (conformance/tests/l0-static/flux-postbuild-substitution). The
|
||||
# percentage test is therefore
|
||||
# a regex over `df -P` output rather than arithmetic on a captured value.
|
||||
- name: docker-prune
|
||||
image: docker:29-cli
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'exit 0' TERM INT
|
||||
echo "docker-prune: started; prunes when /var/lib/docker is at or above 45% used"
|
||||
while true; do
|
||||
if df -P /var/lib/docker | grep -qE ' (4[5-9]|[5-9][0-9]|100)% '; then
|
||||
echo "docker-prune: threshold reached, before:"
|
||||
df -P /var/lib/docker | tail -n 1
|
||||
docker system prune --all --force || echo "docker-prune: prune failed; retrying next cycle"
|
||||
echo "docker-prune: after:"
|
||||
df -P /var/lib/docker | tail -n 1
|
||||
fi
|
||||
sleep 600 &
|
||||
wait
|
||||
done
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://localhost:2376
|
||||
- name: DOCKER_CERT_PATH
|
||||
value: /certs/client
|
||||
- name: DOCKER_TLS_VERIFY
|
||||
value: "1"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: certs
|
||||
mountPath: /certs
|
||||
readOnly: true
|
||||
# Mounted only so `df` can read the volume's usage; the prune itself goes through the
|
||||
# daemon's API.
|
||||
- name: dind-storage
|
||||
mountPath: /var/lib/docker
|
||||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
ephemeral-storage: 16Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
ephemeral-storage: 128Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: act-runner-data
|
||||
- name: certs
|
||||
emptyDir:
|
||||
sizeLimit: 16Mi # a handful of PEM files
|
||||
emptyDir: {}
|
||||
- name: config
|
||||
configMap:
|
||||
name: act-runner-config
|
||||
# The Docker daemon's data root: every job image and job container. OFF the node's /var.
|
||||
#
|
||||
# WHY NOT an emptyDir (what this was until 2026-11-14). kubelet emptyDirs live on /var, a
|
||||
# separate 11.8 GiB partition on every node, and containerd's image store is NOT there
|
||||
# (k3s --data-dir is /data/k3s, infra/ansible/inventory/group_vars/k3s_prod/vars.yml), so
|
||||
# kubelet image GC never frees /var. Measured 2026-11-14: this emptyDir held 3.75 GiB on
|
||||
# k3s-prod-2 (renovate 1.99 GB + toolbox 2.04 GB) with ~1.9 GiB of /var left; the
|
||||
# 2026-11-12 eviction storm was it tipping the node into DiskPressure. A sizeLimit alone
|
||||
# would only trade that for this pod evicting itself, and /var is too small to give the
|
||||
# cache the room it needs next to vuln-scanner's and buildkit's.
|
||||
#
|
||||
# WHY a GENERIC EPHEMERAL volume on `longhorn-scratch` and not a standalone PVC.
|
||||
# * Off /var: Longhorn's dataPath is /data/longhorn on the ~424 GB partition (~170 GB free
|
||||
# per node, measured 2026-11-14).
|
||||
# * Single replica (`longhorn-scratch`, dataLocality best-effort): a disposable cache gains
|
||||
# nothing from 3x replication, and replica writes of multi-GB image pulls would land on
|
||||
# the device etcd's WAL shares (docs/04 accepted risk). Same reasoning as CDI's scratch,
|
||||
# which is what that class exists for.
|
||||
# * Pod-lifetime, not persistent: the claim is created with the pod and deleted with it.
|
||||
# That keeps this out of R-STATE-02 ("persistent storage MUST be replicated"), which a
|
||||
# long-lived single-replica PVC would violate. It also means a replacement pod never
|
||||
# waits on a lone replica stranded on a dead node: it gets a fresh, empty volume. The
|
||||
# cost, accepted: the cache is cold after every pod replacement (~4 GB of pulls), which
|
||||
# is exactly what the emptyDir already did.
|
||||
# * Bounded twice: the claim size is a hard ceiling (ENOSPC for jobs, never a node
|
||||
# eviction), and the docker-prune container keeps it under 45%.
|
||||
# Velero: excluded by the pod annotation above AND the claim label below.
|
||||
- name: dind-storage
|
||||
ephemeral:
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: act-runner
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn-scratch
|
||||
resources:
|
||||
requests:
|
||||
storage: 30Gi
|
||||
emptyDir: {}
|
||||
+29
-1
@@ -60,7 +60,22 @@ env:
|
||||
# ALLOW_ONLY_INTERNAL_REGISTRATION and ENABLE_AUTO_REGISTRATION, not on DISABLE_REGISTRATION.
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: "true"
|
||||
- name: GITEA_ADMIN_USERNAME # break-glass local admin creds (consumed by the postStart hook)
|
||||
# NOTHING IS READABLE ANONYMOUSLY (operator decision 2026-11-14). Since PR #156 put gitea.<domain>
|
||||
# on the public Cloudflare Tunnel, anonymous crawlers kept this pod at 2-6 cores (~24k requests in
|
||||
# 90 min, mostly /<owner>/<repo>/compare/... at ~13 s each). Gitea stays public, but every page,
|
||||
# API call, git clone/fetch and container-registry pull now needs a signed-in user or a token.
|
||||
# `true` is the STRICT mode (setting.Service.RequireSignInViewStrict): it also disables the
|
||||
# anonymous git pull path (routers/web/repo/githttp.go) and the anonymous /v2/token exchange
|
||||
# (routers/api/packages/container/container.go). `expensive` would only hide the costly pages and
|
||||
# leave code readable, which is not what was asked for.
|
||||
# Still anonymous, by upstream design (routers/web/web.go): /robots.txt, /api/healthz, /user/login
|
||||
# and /user/oauth2/keycloak — so humans can still reach the Keycloak button.
|
||||
# EVERY anonymous reader had to gain a credential before this could be turned on; the inventory,
|
||||
# and the operator prerequisite for Flux's own source, are in
|
||||
# decisions/2026-11-14-gitea-require-signin.md. Flipping this back to false is safe at any time.
|
||||
- name: GITEA__service__REQUIRE_SIGNIN_VIEW
|
||||
value: "true"
|
||||
- name: GITEA_ADMIN_USERNAME # break-glass local admin creds (consumed by the postStart hook)
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitea-bootstrap
|
||||
@@ -120,6 +135,19 @@ lifecycle:
|
||||
done
|
||||
echo "[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'"
|
||||
fi
|
||||
# --- robots.txt: ask polite crawlers to stay away entirely (defence in depth) ---
|
||||
# REQUIRE_SIGNIN_VIEW already refuses them, but every refused hit is still a redirect to
|
||||
# /user/login that Gitea has to serve. Gitea serves $GITEA_CUSTOM/public/robots.txt
|
||||
# (/data/gitea in this image), checked on EVERY request (routers/web/misc/misc.go), and
|
||||
# the route is outside the sign-in requirement. Written after the DB wait above, so the
|
||||
# entrypoint has already created and chowned /data/gitea. Rewritten each boot, so this
|
||||
# declaration is the only source of its content.
|
||||
if su-exec git mkdir -p /data/gitea/public \
|
||||
&& printf 'User-agent: *\nDisallow: /\n' | su-exec git tee /data/gitea/public/robots.txt >/dev/null; then
|
||||
echo "[gitea-bootstrap] robots.txt written (Disallow: /)"
|
||||
else
|
||||
echo "[gitea-bootstrap] could not write /data/gitea/public/robots.txt" >&2
|
||||
fi
|
||||
# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---
|
||||
# Registers the confidential `gitea` client as the OAuth2 login source "keycloak" against the
|
||||
# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin
|
||||
|
||||
@@ -242,6 +242,10 @@ patches:
|
||||
# git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git -> refs/heads/main
|
||||
# If the GitRepository ever reports an auth failure that looks like a network problem, RE-CHECK
|
||||
# THAT FIRST: a private repo here fails exactly that way.
|
||||
# SUPERSEDED FOR PROD'S OWN GITEA (2026-11-14): the patch below now carries a secretRef, because
|
||||
# prod's Gitea requires sign-in. If prod is ever re-bootstrapped from the workstation container,
|
||||
# the `flux-gitea-read` token must be valid THERE too (Gitea rejects a bad credential even on a
|
||||
# public repo), or drop the secretRef for the bring-up window.
|
||||
#
|
||||
# *** CUTOVER — the exit condition, mirroring dev's Phase-6 cutover (docs/05) ***
|
||||
# WHEN: Gitea runs on prod (reconciled in-cluster) and answers gitea.home.keithjackson.dev
|
||||
@@ -317,12 +321,28 @@ patches:
|
||||
# domain change cannot sever prod from its own source again.
|
||||
#
|
||||
# Plain http is deliberate and safe here: it is cluster-internal traffic to a ClusterIP,
|
||||
# TLS terminates at the Gateway for external clients only, the GitRepository has no
|
||||
# secretRef (the repo is cloned anonymously), and every secret in the tree is SOPS-encrypted
|
||||
# at rest regardless of transport.
|
||||
# TLS terminates at the Gateway for external clients only, and every secret in the tree is
|
||||
# SOPS-encrypted at rest regardless of transport. The read token below crosses that same
|
||||
# pod-to-ClusterIP hop in a basic-auth header; it is read-only by scope.
|
||||
url: http://gitea.gitea.svc.cluster.local:3000/Home-Network/home-network-k3s-impl.git
|
||||
ref:
|
||||
branch: main
|
||||
# AUTHENTICATED SINCE 2026-11-14 — ANONYMOUS CLONE NO LONGER EXISTS. Gitea now runs with
|
||||
# [service] REQUIRE_SIGNIN_VIEW=true (catalog/gitea.yaml), whose strict mode refuses the
|
||||
# anonymous git pull path outright, public repo or not. Without this secretRef prod cannot
|
||||
# fetch ANY new revision — including the one that would undo this.
|
||||
#
|
||||
# The Secret is OUT-OF-BAND, like `sops-age` beside it, and it has to be: it is the
|
||||
# credential that lets Flux read git at all, so it cannot be delivered BY git (on a fresh
|
||||
# cluster nothing could decrypt or apply it before the first fetch), and the root
|
||||
# Kustomization that applies this file has no SOPS decryption anyway.
|
||||
# keys: username (any value; Gitea resolves the account from the token)
|
||||
# password (a Gitea access token with scope read:repository ONLY)
|
||||
# It MUST exist before this line reaches the cluster: a missing Secret stalls the source
|
||||
# with "secret not found", and the fix is then exactly to create it. Operator steps:
|
||||
# decisions/2026-11-14-gitea-require-signin.md.
|
||||
secretRef:
|
||||
name: flux-gitea-read
|
||||
- target:
|
||||
kind: ConfigMap
|
||||
name: cluster-vars
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
---
|
||||
date: 2026-11-14
|
||||
branch: fix/bound-var-ephemeral-caches
|
||||
title: Bound the three caches filling each node's /var (act-runner off /var, vuln-scanner fitted)
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Each node's `/var` is a separate ~11.6 GiB partition holding kubelet emptyDirs and pod logs;
|
||||
containerd's image store is on `/data/k3s`, so kubelet image GC never frees `/var`. On 2026-11-12
|
||||
the act-runner's unbounded Docker cache tipped k3s-prod-2 into DiskPressure (63 act-runner
|
||||
evictions).
|
||||
|
||||
- `apps/wave-1/gitea-actions.yaml` (hand-authored: a renderer `EXTRA_UNITS` file, placed by
|
||||
`catalog/render.py` but never generated, so no re-render was needed or run):
|
||||
- `dind-storage` is now a **generic ephemeral volume** (30Gi, `longhorn-scratch`) instead of an
|
||||
unbounded `emptyDir {}`. Off `/var`, onto `/data/longhorn`.
|
||||
- New `docker-prune` container (`docker:29-cli`): every 10 min, if the volume is >=45% used,
|
||||
`docker system prune --all --force`. Threshold-based, because image `until=` filters use image
|
||||
creation time and would re-pull long-lived job images between jobs. No `$` in the script
|
||||
(Flux postBuild substitution runs over the file).
|
||||
- ephemeral-storage requests/limits on dind (256Mi/2Gi), runner (128Mi/1Gi) and prune
|
||||
(16Mi/128Mi); `certs` emptyDir `sizeLimit: 16Mi`.
|
||||
- Velero opt-out: pod annotation `backup.velero.io/backup-volumes-excludes: dind-storage` and
|
||||
the claim label `velero.io/exclude-from-backup: "true"`. Daily schedule is opt-out and its
|
||||
volume policy skips only emptyDir/NFS, so without this the cache would go to R2 nightly.
|
||||
- `platform/vuln-scanner/deployment.yaml`: `/cache` emptyDir sizeLimit 10Gi -> 5Gi; scanner
|
||||
ephemeral-storage request 256Mi -> 3584Mi (measured steady state), limit 12Gi -> 6Gi (covers
|
||||
cache + tools + logs, since kubelet compares pod usage with the sum of container limits). Added
|
||||
soft pod anti-affinity (weight 100) from `buildkit/buildkitd`. The prod-2 node preference is kept:
|
||||
with the runner off `/var`, sharing prod-2 no longer stacks caches.
|
||||
- buildkit: unchanged (maxUsedSpace 4GB, sizeLimit 6Gi, bounded). It has no anti-affinity back to
|
||||
the scanner; the scanner side alone carries it.
|
||||
|
||||
### Why not what the brief preferred
|
||||
|
||||
- **Persistent PVC for the runner cache** (survives pod moves): not done. A long-lived
|
||||
single-replica PVC is persistent storage that is not replicated, against R-STATE-02. A 3-replica
|
||||
PVC would triple multi-GB pull writes onto the device etcd's WAL shares. A generic ephemeral
|
||||
volume is pod-lifetime, the same class as the emptyDir it replaces. It is also never stranded on a
|
||||
dead node: a replacement pod gets a fresh claim. Cost: the cache is cold after a pod replacement,
|
||||
which is what the emptyDir already did.
|
||||
- **vuln-scanner cache onto a PVC**: not done. docs/17 "Load" says scan caches MUST NOT be on a
|
||||
replicated Longhorn volume *because etcd and Longhorn share a partition*. A single-replica volume
|
||||
meets the letter but not the reason. L0 `vuln-scanner-supply-chain` also requires the cache to be
|
||||
a size-limited emptyDir.
|
||||
|
||||
### Spec escalations (for the architect, not acted on)
|
||||
|
||||
1. docs/04 calls CDI scratch "the one non-replicated use of Longhorn". `longhorn-scratch` now also
|
||||
backs the act-runner's pod-lifetime Docker cache. The sentence should say "throwaway,
|
||||
pod-lifetime scratch", or name this use.
|
||||
2. docs/04's rebuild rule and R-STATE-02 don't say whether a generic ephemeral volume counts as
|
||||
"persistent storage". This change reads it as not.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Live, read-only, 2026-11-14 (kubelet `stats/summary`): nodefs capacity/available prod-1
|
||||
11.60/8.89 GiB, prod-2 11.85/1.93, prod-3 11.60/4.75. prod-2 `dind-storage` 3.75 GiB, vuln
|
||||
`cache` 3.12 GiB; prod-3 buildkit `cache` 3.86 GiB. Non-cache `/var` use ~2.1-2.5 GiB per node.
|
||||
- StorageClasses live: `longhorn-scratch` = `numberOfReplicas: "1"`, `dataLocality: best-effort`,
|
||||
reclaim Delete. Longhorn disk available per node 177-187 GB; over-provisioning 200%, minimal
|
||||
available 25%.
|
||||
- Node allocatable ephemeral-storage 11.8-12.1 GB; current requests prod-2 256Mi, prod-3 2Gi, so the
|
||||
3584Mi scanner request schedules.
|
||||
- No Kyverno policy selects the non-app `gitea-actions` namespace for volume shape. None of the
|
||||
touched paths are in `infra/gitea/approval-policy.json` `sensitive_paths`.
|
||||
- Toolbox gate from this worktree (DEV_KUBECONFIG=/dev/null, ENGINE shim mounting the main
|
||||
`.git`, spec repo mounted by dev.sh as a sibling), on the committed HEAD: **rc 3, UNDETERMINED,
|
||||
not green.** "Stages that did not run: gitea-settings". `conformance` (L2/L3) logged SKIPPED, no
|
||||
cluster. spec-immutability, approval-policy (N/A offline, proves nothing), decisions-log,
|
||||
lint, l0 (incl. `flux-postbuild-substitution` and `vuln-scanner-supply-chain` PASS),
|
||||
render-golden and l1 green.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
Nothing here has been deployed. This is only provable after merge, under Flux.
|
||||
|
||||
- That the generic ephemeral claim provisions and attaches on `longhorn-scratch`, and that dind
|
||||
starts on it within apps-wave-1's 5m health timeout (`wait: true`). If it doesn't, wave 1 stalls,
|
||||
and with it every wave above it.
|
||||
- That `docker system prune` runs: TLS to dind from the sidecar, BusyBox `df -P` output matching
|
||||
the regex, cert file permissions readable by a capability-less root process. UNVERIFIED.
|
||||
- That Velero skips the volume and the claim. The annotation and label are documented Velero
|
||||
opt-outs but were not exercised here.
|
||||
- That 5Gi is enough for the largest image Trivy scans. Peak extraction size was not measured. An
|
||||
overrun evicts only the scanner, and the image sits out per the in-flight marker.
|
||||
- `runbooks/node-disk.md` still points at `/var/lib/rancher/k3s` for containerd. That is stale
|
||||
(it's on `/data/k3s`) and not fixed here.
|
||||
@@ -0,0 +1,161 @@
|
||||
---
|
||||
date: 2026-11-14
|
||||
branch: feat/gitea-require-signin
|
||||
title: Gitea requires sign-in to read anything; Flux and CI authenticate first
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Operator decision (2026-11-14): Gitea stays on the public internet, but reading anything needs a
|
||||
login. Since PR #156 put `gitea.<domain>` on the Cloudflare Tunnel wildcard, anonymous crawlers had
|
||||
kept the Gitea pod at 2–6 cores. The traffic was ~24k requests in 90 minutes from two IPs, mostly
|
||||
`/<owner>/<repo>/compare/...?style=split` at ~13 s each.
|
||||
|
||||
- **`catalog/gitea.yaml` → `apps/wave-0/gitea.yaml`** (re-rendered with the canonical command; only
|
||||
`gitea.yaml` differs from the committed tree):
|
||||
- `GITEA__service__REQUIRE_SIGNIN_VIEW=true`. This is the strict mode
|
||||
(`RequireSignInViewStrict`). It refuses anonymous web pages, API calls, smart-HTTP git pulls
|
||||
(`routers/web/repo/githttp.go`) and the anonymous container-registry `/v2/token` exchange
|
||||
(`routers/api/packages/container/container.go`). `expensive` was not chosen because it leaves
|
||||
code readable.
|
||||
- The postStart hook writes `/data/gitea/public/robots.txt` (`Disallow: /`) on every boot. Gitea
|
||||
checks for that file on every request, and `/robots.txt` is routed outside the sign-in
|
||||
requirement.
|
||||
- The login path is unchanged. Keycloak was already the native OIDC source `keycloak`, registered
|
||||
by the same hook with auto-registration on. Local registration was already disabled. The local
|
||||
break-glass admin stays. OpenID 2.0 sign-in and sign-up are already off by upstream default.
|
||||
Basic auth stays at its default (on); disabling it would only stop password basic auth, not
|
||||
tokens.
|
||||
- **`clusters/prod/kustomization.yaml`**: the `flux-system` GitRepository gains
|
||||
`secretRef: flux-gitea-read`. This Secret is created out-of-band, like `sops-age`. It cannot come
|
||||
from git: it is the credential Flux needs to read git at all, and the root Kustomization has no
|
||||
SOPS decryption.
|
||||
- **`.gitea/workflows/gate.yml`, `.gitea/workflows/cve-triage.yaml`**: the job container
|
||||
(`gitea.../home-network/toolbox@sha256:…`) was pulled anonymously. It now declares
|
||||
`container.credentials` with the existing `RENOVATE_TOKEN`. renovate.yaml's preflight proves
|
||||
that token's `read:package` scope on every run, and cve-triage already holds it. act fails the job
|
||||
if either field interpolates empty, so a missing secret shows up as red rather than as a silent
|
||||
fallback.
|
||||
|
||||
### Anonymous-reader inventory (what would break, and how each is handled)
|
||||
|
||||
Live state was measured read-only on 2026-11-14. Anonymous `GET /api/v1/repos/...impl` returned
|
||||
200, as did anonymous `/v2/token` for toolbox. `/robots.txt` returned 404. Both repos
|
||||
(`home-network-k3s`, `-impl`) are public.
|
||||
|
||||
| Reader | How it reads today | Handling |
|
||||
|---|---|---|
|
||||
| Flux `GitRepository flux-system` | **anonymous** `http://gitea.gitea.svc.cluster.local:3000/...` (live: no `secretRef`; `flux-system` holds only `sops-age`) | `secretRef: flux-gitea-read`. **Needs a NEW token: OPERATOR PREREQUISITE, see below** |
|
||||
| gate.yml job container (toolbox) | **anonymous** registry pull by the runner's DinD | `credentials:` with existing `RENOVATE_TOKEN` (this PR) |
|
||||
| cve-triage.yaml job container (toolbox) | **anonymous** registry pull | same (this PR) |
|
||||
| `actions/checkout` / gate's git fetch | Actions job token for its own repo | unaffected (signed-in actions user, same repo) |
|
||||
| merge-gate.yaml cross-repo spec checkout | Actions job token, `vars.SPEC_REPO_SLUG` | dormant (`workflow_dispatch` only, slug unwired). Whether the job token can read *another* repo under strict mode is UNVERIFIED |
|
||||
| images.yaml | PR: builds toolbox in-run (no pull). main: `REGISTRY_PUSH_TOKEN` in buildx auth and `/v2/token` | unaffected |
|
||||
| Renovate | `RENOVATE_TOKEN` for API, git and docker hostRules | unaffected |
|
||||
| pm-agent dispatcher and workers | `GITEA_TOKEN` (API, GIT_ASKPASS clone); image via `imagePullSecrets: pm-agent-registry` | unaffected |
|
||||
| vuln-scanner | does not pull the Gitea registry: `VULN_PRIVATE_REGISTRIES=gitea.${BASE_DOMAIN}` short-circuits to `credential-required` | unaffected |
|
||||
| act_runner | registration and runner token | unaffected; `DEFAULT_ACTIONS_URL=github`, so no action is fetched from Gitea |
|
||||
| kubelet/containerd image pulls | live: the only Gitea-registry image is pm-agent's, which has a pull secret; no `registries.yaml` in infra/ansible | unaffected |
|
||||
| Flux image-reflector/automation | live: zero ImageRepository/ImagePolicy/ImageUpdateAutomation objects | n/a |
|
||||
| conformance in-cluster clone (`conformance/incluster/conformance-clone`) | anonymous unless `conformance-git-readonly` is set | dormant (L2/L3 not wired). **When it is wired, the read-only Secret is now mandatory, not optional** |
|
||||
| ci/gitea-settings.sh, ci/approval-policy.sh, infra/gitea/*.sh | `SETTINGS_API_TOKEN` and other tokens | unaffected |
|
||||
| Homepage | a plain `href`, no widget API call | unaffected (the browser hits the login page) |
|
||||
| Prometheus / probes | Gitea has no probes and no ServiceMonitor; `[metrics]` is not enabled | unaffected |
|
||||
| Operator and agent workstations | git credential helper (both repos) | unaffected |
|
||||
| Public spec docs linking to the impl repo | anonymous browsers | now land on the login page (see spec conflict below) |
|
||||
|
||||
### OPERATOR STEPS — in this order, BEFORE the merge
|
||||
|
||||
1. Mint a token with **scope `read:repository` only**. Use a dedicated non-admin local account
|
||||
(e.g. `flux-reader`, no org membership needed: with sign-in required, any signed-in user can
|
||||
read a public repo). The existing non-admin `renovate-bot` also works. Do not use an admin
|
||||
account.
|
||||
2. Create the Secret out-of-band, without the token reaching shell history or a transcript:
|
||||
|
||||
read -rs TOKEN; export TOKEN
|
||||
./infra/devcontainer/dev.sh run bash -c "kubectl --context k3s-prod -n flux-system \
|
||||
create secret generic flux-gitea-read --from-literal=username=flux-reader \
|
||||
--from-literal=password=$TOKEN"
|
||||
|
||||
(`$TOKEN` expands on the host into the container's argv, which is briefly visible in `ps`. If
|
||||
that matters, use `--from-file` on a 0600 file and delete the file.)
|
||||
3. Prove it before merging, in the same shell, then `unset TOKEN`. First check that the token can clone. This command lists `refs/heads/main`, and the token stays
|
||||
out of argv:
|
||||
|
||||
git -c credential.helper='!f(){ echo username=x; echo "password=$TOKEN"; };f' \
|
||||
ls-remote https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git main
|
||||
|
||||
Then check that the Secret has the right shape. This prints key names only and should show
|
||||
`password username`:
|
||||
|
||||
kubectl --context k3s-prod -n flux-system get secret flux-gitea-read \
|
||||
-o go-template='{{range $k,$v := .data}}{{$k}} {{end}}'
|
||||
4. Post `/approve-override <reason>` as a PR review. This PR touches sensitive paths (`clusters/**`,
|
||||
`.gitea/**`); the author must not post it.
|
||||
5. Merge. Then confirm `kubectl -n flux-system get gitrepository flux-system` is Ready on the merge
|
||||
SHA, and that anonymous `curl https://gitea.home.keithjackson.dev/api/v1/repos/Home-Network/home-network-k3s-impl`
|
||||
now returns 401/403. Also check that `/robots.txt` returns 200.
|
||||
|
||||
**If step 2 is skipped**, the source goes `False` with "secret not found" right after the merge and
|
||||
prod stops taking new revisions. That includes any revert. Recovery is the same as step 2: create
|
||||
the Secret. Nothing else needs undoing. Between steps 2 and 5 the token is harmless: an
|
||||
authenticated read of a public repo works exactly as the anonymous one did.
|
||||
|
||||
### Spec conflicts (for the architect; the spec repo was not modified)
|
||||
|
||||
- **ADR-0027** rejects a CVE-findings issue *because "the repository is publicly readable and the
|
||||
list would be an attack map"*. That premise is now false for anonymous readers. Any signed-in
|
||||
Keycloak user can still read it (see the next point), so the rejection arguably still stands, but
|
||||
the stated reason needs updating.
|
||||
- **docs/15, R-IDP-06** ("each gate … enforces the service's `auth.groups`"): Gitea's catalog
|
||||
declares `auth.groups: [platform-admins]`, but the `keycloak` auth source only maps that group to
|
||||
*admin*. It does not *require* the group. With sign-in required, "read" therefore means "any user
|
||||
in the homelab realm", which matches what the operator asked for ("a Keycloak login"). Restricting
|
||||
it further needs `--required-claim-name groups --required-claim-value platform-admins` on the auth
|
||||
source. That is not done here, because it could lock out non-admin realm users who use Gitea today.
|
||||
It is an architect/operator call.
|
||||
- **docs/05 and docs/09** link to the impl repo as a public URL; those links now land on a login
|
||||
page.
|
||||
|
||||
### Sensitive paths
|
||||
|
||||
`clusters/prod/kustomization.yaml` (`clusters/**`) and `.gitea/workflows/*` (`.gitea/**`) are in
|
||||
`infra/gitea/approval-policy.json`, so CI's approval-policy stage blocks until the operator posts the
|
||||
override.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Toolbox gate run from the worktree on the committed tree: **rc 3 (UNDETERMINED, not green)**. The
|
||||
gate printed "Stages that did not run: gitea-settings". That stage needs the CI-only
|
||||
`SETTINGS_API_TOKEN`. `spec-immutability` passed with the sibling spec repo mounted. `lint`, `l0`,
|
||||
`render-golden`, `l1` and `decisions-log` passed. Conformance L2/L3 SKIPPED (no cluster). It
|
||||
lists R-IDP-02/06, R-GITOPS-01/02 and others as UNVERIFIED. `approval-policy` is N/A offline and
|
||||
will block in CI until the override is posted.
|
||||
- The canonical render (`render.py --all catalog -o <tmp> --wave-layout --no-image-automation`) differs
|
||||
from the committed `apps/` in `wave-0/gitea.yaml` only, and only by the new env var and the hook
|
||||
text.
|
||||
- Live, read-only: the GitRepository has no secretRef, and `flux-system` has only `sops-age`. Zero
|
||||
image-automation objects exist. The only pod pulling from the Gitea registry is pm-agent, and it
|
||||
has a pull secret. Anonymous API and `/v2/token` both return 200 today, and `/robots.txt` returns
|
||||
404.
|
||||
- From upstream Gitea source on `main` (not the 28.3.0 tag): strict mode gates githttp anonymous
|
||||
pull, `ReqContainerAccess` and `/v2/token` `Authenticate`. `/robots.txt`, `/api/healthz`,
|
||||
`/user/login` and `/user/oauth2/{provider}` are registered outside sign-in. robots.txt is read
|
||||
from `CustomPath/public/robots.txt` on each request.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **Nothing is deployed.** No live Gitea has run with `REQUIRE_SIGNIN_VIEW=true`, and none of the
|
||||
claims above has been observed at runtime.
|
||||
- The upstream code reading is of `main`, not the 28.3.0 tag that runs here.
|
||||
- That `container.credentials` actually authenticates on this act_runner (0.6.1). The PR's own CI run
|
||||
proves the syntax is accepted and the secret interpolates. While the image is cached in DinD
|
||||
(`force_pull` false), no pull happens, so the auth path is only exercised on a cold runner.
|
||||
- That `RENOVATE_TOKEN` is currently valid. Its last renovate.yaml preflight was not inspected.
|
||||
- That the Flux token works. It does not exist yet; that is an operator step.
|
||||
- Whether an Actions job token can read a *different* repo under strict mode. It only matters for the
|
||||
dormant merge-gate.yaml spec checkout.
|
||||
- That the robots.txt write succeeds (busybox `tee` via `su-exec git` into `/data/gitea/public`).
|
||||
The hook logs either outcome and cannot crash the pod.
|
||||
- Crawler load actually dropping after the change. Compare the pod's CPU on k3s-prod-2 before and
|
||||
after.
|
||||
@@ -14,15 +14,6 @@
|
||||
#
|
||||
# SOFT affinity toward k3s-prod-2: prod-1 is the node whose etcd I/O starved historically, and scans
|
||||
# are CPU- and I/O-heavy. Soft on purpose: a scanner that cannot be placed must still run elsewhere.
|
||||
#
|
||||
# SOFT anti-affinity from buildkitd (added 2026-11-14). The /cache emptyDir lives on the node's /var,
|
||||
# a separate ~11.6 GiB partition kubelet evicts from at <5% free, and buildkitd's own /var cache is
|
||||
# bounded at 6Gi. Measured 2026-11-14: ~2.2-2.5 GiB of /var is already used on each node by
|
||||
# everything else, so this cache (<=5Gi) and buildkit's (<=6Gi) cannot both fit on one node. The
|
||||
# third big /var tenant, the act-runner Docker cache, moved off /var the same day
|
||||
# (apps/wave-1/gitea-actions.yaml, `dind-storage`), so sharing prod-2 with Gitea and the runner no
|
||||
# longer stacks caches and the prod-2 preference stays. Weight 100 on both: when prod-2 is
|
||||
# unavailable the scanner prefers prod-1 over buildkit's node. Soft, for the same reason as above.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -70,16 +61,6 @@ spec:
|
||||
operator: In
|
||||
values:
|
||||
- k3s-prod-2
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
namespaces:
|
||||
- buildkit
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: buildkitd
|
||||
initContainers:
|
||||
- name: trivy-tools
|
||||
# TRIVY 0.76.1 — pinned by digest (R-VULN-09). Not 0.69.4/5/6 (compromised release, March
|
||||
@@ -167,24 +148,15 @@ spec:
|
||||
- { name: metrics, containerPort: 9090, protocol: TCP }
|
||||
# Sized to trivy, the only heavy thing here: scans are sequential, the limit is the peak of
|
||||
# one large image; the emptyDir sizeLimit bounds disk.
|
||||
#
|
||||
# ephemeral-storage, RESIZED 2026-11-14. It was request 256Mi / limit 12Gi with a 10Gi
|
||||
# cache sizeLimit, on a node /var of ~11.6 GiB: neither bound could fire before kubelet's
|
||||
# node-level DiskPressure eviction did, which takes other pods down with it. kubelet
|
||||
# compares the POD's usage (its emptyDirs + container writable layers + logs) against the
|
||||
# SUM of its containers' limits, and only this container carries one, so this limit must
|
||||
# cover cache (5Gi) + tools (512Mi) + logs: 6Gi. The request is the measured steady state
|
||||
# (3.12 GiB of cache: Trivy DB 1.7G + Java DB 1.5G, 2026-11-14) so the scheduler stops
|
||||
# treating this pod as ~free of local disk.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
ephemeral-storage: 3584Mi
|
||||
ephemeral-storage: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
ephemeral-storage: 6Gi
|
||||
ephemeral-storage: 12Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
@@ -308,13 +280,6 @@ spec:
|
||||
claimName: vuln-scanner-data
|
||||
# Trivy DB + per-image scan cache + TMPDIR (trivy extracts layers here). Size-limited; never
|
||||
# on a replicated volume or in etcd (docs/17 "Load"). A full run cleans it between images.
|
||||
#
|
||||
# 5Gi, down from 10Gi (2026-11-14): 10Gi was most of the node's whole ~11.6 GiB /var, so it
|
||||
# could never fire before node DiskPressure. 5Gi = the 3.12 GiB of DBs measured 2026-11-14
|
||||
# plus ~1.9 GiB for one image's extraction (TMPDIR points here, so extraction is inside the
|
||||
# bound) and a DB refresh. Overrunning it evicts THIS pod only; main.py records the
|
||||
# in-flight image as failed and sits it out, so an oversized image cannot loop the scanner.
|
||||
# If the DBs grow past ~4 GiB, raise this AND the scanner's ephemeral-storage limit together.
|
||||
- name: cache
|
||||
emptyDir:
|
||||
sizeLimit: 5Gi
|
||||
sizeLimit: 10Gi
|
||||
Reference in new issue
Block a user