From 3ae7fb74f8d4900527f0e56026e033bff588c384 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 14 Nov 2026 14:15:00 -0600 Subject: [PATCH 1/2] fix(storage): bound the caches filling each node's /var The act-runner's Docker cache was an unbounded emptyDir on the 11.6 GiB /var partition and evicted k3s-prod-2 on 2026-11-12. It moves to a generic ephemeral volume on longhorn-scratch (off /var, pod-lifetime, single replica), with a threshold-triggered docker prune and Velero opt-out. vuln-scanner's cache sizeLimit and ephemeral-storage limit are cut to fit the partition, and it prefers not to share a node with buildkitd. Co-Authored-By: Claude Opus 5.5 --- apps/wave-1/gitea-actions.yaml | 133 +++++++++++++++++- .../2026-11-14-bound-var-ephemeral-caches.md | 84 +++++++++++ platform/vuln-scanner/deployment.yaml | 41 +++++- 3 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 decisions/2026-11-14-bound-var-ephemeral-caches.md diff --git a/apps/wave-1/gitea-actions.yaml b/apps/wave-1/gitea-actions.yaml index 6887105..63d5d81 100644 --- a/apps/wave-1/gitea-actions.yaml +++ b/apps/wave-1/gitea-actions.yaml @@ -135,6 +135,13 @@ spec: # Bump when act-runner-config changes without any other pod-template change, so the # Deployment rolls and act_runner re-reads its config (see the ConfigMap header). homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2 + # Velero's daily schedule is opt-OUT (defaultVolumesToFsBackup: true) and its volume policy + # skips only emptyDir and NFS (platform/velero/volume-policies.yaml). dind-storage is now a + # Longhorn-backed generic ephemeral volume (see `volumes:`), which that policy does NOT + # match, so without this the nightly backup would push the whole Docker image cache to R2 + # and race this pod's replacement (the PartiallyFailed failure that file documents). It is a + # disposable cache: nothing in it is state. + backup.velero.io/backup-volumes-excludes: dind-storage spec: # CI holds no cluster credentials — do not mount a SA token into the runner or # (via the shared netns) the job containers (R-SEC-02, docs/14). @@ -197,12 +204,18 @@ spec: # memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here # takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom # over that for the parallelised gate. Nodes have ~31Gi allocatable. + # ephemeral-storage: /var/lib/docker (images, job containers, their logs) is on the + # dind-storage PVC, NOT the node's /var, so this bounds only dockerd's own rootfs + # writes and its container log. Before 2026-11-14 the image cache was an unbounded + # emptyDir on /var (see `volumes:`). resources: requests: cpu: "2" memory: 1Gi + ephemeral-storage: 256Mi limits: memory: 4Gi + ephemeral-storage: 2Gi startupProbe: exec: command: @@ -266,16 +279,132 @@ spec: requests: cpu: 100m memory: 256Mi + ephemeral-storage: 128Mi limits: memory: 1Gi + ephemeral-storage: 1Gi + # ---- docker-prune: keeps the dind image cache bounded ---------------------------------- + # + # dockerd never garbage-collects images. Every new renovate/toolbox digest (~2 GB each) used + # to stack up in /var/lib/docker until the node's /var filled: on 2026-11-12 that tipped + # k3s-prod-2 into DiskPressure and the act-runner pod was evicted 63 times. Moving the cache + # to its own volume (see `volumes:`) stops it hurting the node; this stops it filling that + # volume and failing jobs with ENOSPC instead. + # + # Every 10 minutes: if the volume is at or above 45% used, `docker system prune --all + # --force` removes every image, stopped container, unused network and build cache entry NOT + # in use by a container. Images a running job uses are never removed. 45% (not 85%) so the + # volume stays above the 50%-free gate of PersistentVolumeFillingUpPredicted + # (platform/monitoring/pvc-capacity-alerts.yaml) and a normal refill never pages anyone. + # Threshold-triggered, not age-based: `--filter until=` on images is image CREATION time, so + # an age-based prune would delete the long-lived job images (catthehacker, the toolbox) between + # every pair of jobs and force a re-pull each time. + # + # Accepted race: a prune landing in the milliseconds between a job's image pull and its + # container create removes that image and fails that one job. It needs the volume over the + # threshold AT that instant; a re-run passes. + # + # NO DOLLAR SIGNS, deliberately: clusters/base/apps-wave-1.yaml runs Flux postBuild + # substitution over this whole file, comments included, which rewrites shell variables and + # command substitutions (conformance/tests/l0-static/flux-postbuild-substitution). The + # percentage test is therefore + # a regex over `df -P` output rather than arithmetic on a captured value. + - name: docker-prune + image: docker:29-cli + command: + - /bin/sh + - -c + - | + trap 'exit 0' TERM INT + echo "docker-prune: started; prunes when /var/lib/docker is at or above 45% used" + while true; do + if df -P /var/lib/docker | grep -qE ' (4[5-9]|[5-9][0-9]|100)% '; then + echo "docker-prune: threshold reached, before:" + df -P /var/lib/docker | tail -n 1 + docker system prune --all --force || echo "docker-prune: prune failed; retrying next cycle" + echo "docker-prune: after:" + df -P /var/lib/docker | tail -n 1 + fi + sleep 600 & + wait + done + env: + - name: DOCKER_HOST + value: tcp://localhost:2376 + - name: DOCKER_CERT_PATH + value: /certs/client + - name: DOCKER_TLS_VERIFY + value: "1" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: certs + mountPath: /certs + readOnly: true + # Mounted only so `df` can read the volume's usage; the prune itself goes through the + # daemon's API. + - name: dind-storage + mountPath: /var/lib/docker + readOnly: true + resources: + requests: + cpu: 10m + memory: 32Mi + ephemeral-storage: 16Mi + limits: + memory: 128Mi + ephemeral-storage: 128Mi volumes: - name: data persistentVolumeClaim: claimName: act-runner-data - name: certs - emptyDir: {} + emptyDir: + sizeLimit: 16Mi # a handful of PEM files - name: config configMap: name: act-runner-config + # The Docker daemon's data root: every job image and job container. OFF the node's /var. + # + # WHY NOT an emptyDir (what this was until 2026-11-14). kubelet emptyDirs live on /var, a + # separate 11.8 GiB partition on every node, and containerd's image store is NOT there + # (k3s --data-dir is /data/k3s, infra/ansible/inventory/group_vars/k3s_prod/vars.yml), so + # kubelet image GC never frees /var. Measured 2026-11-14: this emptyDir held 3.75 GiB on + # k3s-prod-2 (renovate 1.99 GB + toolbox 2.04 GB) with ~1.9 GiB of /var left; the + # 2026-11-12 eviction storm was it tipping the node into DiskPressure. A sizeLimit alone + # would only trade that for this pod evicting itself, and /var is too small to give the + # cache the room it needs next to vuln-scanner's and buildkit's. + # + # WHY a GENERIC EPHEMERAL volume on `longhorn-scratch` and not a standalone PVC. + # * Off /var: Longhorn's dataPath is /data/longhorn on the ~424 GB partition (~170 GB free + # per node, measured 2026-11-14). + # * Single replica (`longhorn-scratch`, dataLocality best-effort): a disposable cache gains + # nothing from 3x replication, and replica writes of multi-GB image pulls would land on + # the device etcd's WAL shares (docs/04 accepted risk). Same reasoning as CDI's scratch, + # which is what that class exists for. + # * Pod-lifetime, not persistent: the claim is created with the pod and deleted with it. + # That keeps this out of R-STATE-02 ("persistent storage MUST be replicated"), which a + # long-lived single-replica PVC would violate. It also means a replacement pod never + # waits on a lone replica stranded on a dead node: it gets a fresh, empty volume. The + # cost, accepted: the cache is cold after every pod replacement (~4 GB of pulls), which + # is exactly what the emptyDir already did. + # * Bounded twice: the claim size is a hard ceiling (ENOSPC for jobs, never a node + # eviction), and the docker-prune container keeps it under 45%. + # Velero: excluded by the pod annotation above AND the claim label below. - name: dind-storage - emptyDir: {} + ephemeral: + volumeClaimTemplate: + metadata: + labels: + app.kubernetes.io/name: act-runner + velero.io/exclude-from-backup: "true" + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-scratch + resources: + requests: + storage: 30Gi diff --git a/decisions/2026-11-14-bound-var-ephemeral-caches.md b/decisions/2026-11-14-bound-var-ephemeral-caches.md new file mode 100644 index 0000000..f028021 --- /dev/null +++ b/decisions/2026-11-14-bound-var-ephemeral-caches.md @@ -0,0 +1,84 @@ +--- +date: 2026-11-14 +branch: fix/bound-var-ephemeral-caches +title: Bound the three caches filling each node's /var (act-runner off /var, vuln-scanner fitted) +--- + +## What changed + +Each node's `/var` is a separate ~11.6 GiB partition holding kubelet emptyDirs and pod logs; +containerd's image store is on `/data/k3s`, so kubelet image GC never frees `/var`. On 2026-11-12 +the act-runner's unbounded Docker cache tipped k3s-prod-2 into DiskPressure (63 act-runner +evictions). + +- `apps/wave-1/gitea-actions.yaml` (hand-authored: a renderer `EXTRA_UNITS` file, placed by + `catalog/render.py` but never generated, so no re-render was needed or run): + - `dind-storage` is now a **generic ephemeral volume** (30Gi, `longhorn-scratch`) instead of an + unbounded `emptyDir {}`. Off `/var`, onto `/data/longhorn`. + - New `docker-prune` container (`docker:29-cli`): every 10 min, if the volume is >=45% used, + `docker system prune --all --force`. Threshold-based, because image `until=` filters use image + creation time and would re-pull long-lived job images between jobs. No `$` in the script + (Flux postBuild substitution runs over the file). + - ephemeral-storage requests/limits on dind (256Mi/2Gi), runner (128Mi/1Gi) and prune + (16Mi/128Mi); `certs` emptyDir `sizeLimit: 16Mi`. + - Velero opt-out: pod annotation `backup.velero.io/backup-volumes-excludes: dind-storage` and + the claim label `velero.io/exclude-from-backup: "true"`. Daily schedule is opt-out and its + volume policy skips only emptyDir/NFS, so without this the cache would go to R2 nightly. +- `platform/vuln-scanner/deployment.yaml`: `/cache` emptyDir sizeLimit 10Gi -> 5Gi; scanner + ephemeral-storage request 256Mi -> 3584Mi (measured steady state), limit 12Gi -> 6Gi (covers + cache + tools + logs, since kubelet compares pod usage with the sum of container limits). Added + soft pod anti-affinity (weight 100) from `buildkit/buildkitd`. The prod-2 node preference is kept: + with the runner off `/var`, sharing prod-2 no longer stacks caches. +- buildkit: unchanged (maxUsedSpace 4GB, sizeLimit 6Gi, bounded). It has no anti-affinity back to + the scanner; the scanner side alone carries it. + +### Why not what the brief preferred + +- **Persistent PVC for the runner cache** (survives pod moves): not done. A long-lived + single-replica PVC is persistent storage that is not replicated, against R-STATE-02. A 3-replica + PVC would triple multi-GB pull writes onto the device etcd's WAL shares. A generic ephemeral + volume is pod-lifetime, the same class as the emptyDir it replaces. It is also never stranded on a + dead node: a replacement pod gets a fresh claim. Cost: the cache is cold after a pod replacement, + which is what the emptyDir already did. +- **vuln-scanner cache onto a PVC**: not done. docs/17 "Load" says scan caches MUST NOT be on a + replicated Longhorn volume *because etcd and Longhorn share a partition*. A single-replica volume + meets the letter but not the reason. L0 `vuln-scanner-supply-chain` also requires the cache to be + a size-limited emptyDir. + +### Spec escalations (for the architect, not acted on) + +1. docs/04 calls CDI scratch "the one non-replicated use of Longhorn". `longhorn-scratch` now also + backs the act-runner's pod-lifetime Docker cache. The sentence should say "throwaway, + pod-lifetime scratch", or name this use. +2. docs/04's rebuild rule and R-STATE-02 don't say whether a generic ephemeral volume counts as + "persistent storage". This change reads it as not. + +## What I verified + +- Live, read-only, 2026-11-14 (kubelet `stats/summary`): nodefs capacity/available prod-1 + 11.60/8.89 GiB, prod-2 11.85/1.93, prod-3 11.60/4.75. prod-2 `dind-storage` 3.75 GiB, vuln + `cache` 3.12 GiB; prod-3 buildkit `cache` 3.86 GiB. Non-cache `/var` use ~2.1-2.5 GiB per node. +- StorageClasses live: `longhorn-scratch` = `numberOfReplicas: "1"`, `dataLocality: best-effort`, + reclaim Delete. Longhorn disk available per node 177-187 GB; over-provisioning 200%, minimal + available 25%. +- Node allocatable ephemeral-storage 11.8-12.1 GB; current requests prod-2 256Mi, prod-3 2Gi, so the + 3584Mi scanner request schedules. +- No Kyverno policy selects the non-app `gitea-actions` namespace for volume shape. None of the + touched paths are in `infra/gitea/approval-policy.json` `sensitive_paths`. +- Toolbox gate: see the PR description for the exact rc and stages that did not run. + +## What I did NOT verify + +Nothing here has been deployed. This is only provable after merge, under Flux. + +- That the generic ephemeral claim provisions and attaches on `longhorn-scratch`, and that dind + starts on it within apps-wave-1's 5m health timeout (`wait: true`). If it doesn't, wave 1 stalls, + and with it every wave above it. +- That `docker system prune` runs: TLS to dind from the sidecar, BusyBox `df -P` output matching + the regex, cert file permissions readable by a capability-less root process. UNVERIFIED. +- That Velero skips the volume and the claim. The annotation and label are documented Velero + opt-outs but were not exercised here. +- That 5Gi is enough for the largest image Trivy scans. Peak extraction size was not measured. An + overrun evicts only the scanner, and the image sits out per the in-flight marker. +- `runbooks/node-disk.md` still points at `/var/lib/rancher/k3s` for containerd. That is stale + (it's on `/data/k3s`) and not fixed here. diff --git a/platform/vuln-scanner/deployment.yaml b/platform/vuln-scanner/deployment.yaml index 094908c..c7c6498 100644 --- a/platform/vuln-scanner/deployment.yaml +++ b/platform/vuln-scanner/deployment.yaml @@ -14,6 +14,15 @@ # # SOFT affinity toward k3s-prod-2: prod-1 is the node whose etcd I/O starved historically, and scans # are CPU- and I/O-heavy. Soft on purpose: a scanner that cannot be placed must still run elsewhere. +# +# SOFT anti-affinity from buildkitd (added 2026-11-14). The /cache emptyDir lives on the node's /var, +# a separate ~11.6 GiB partition kubelet evicts from at <5% free, and buildkitd's own /var cache is +# bounded at 6Gi. Measured 2026-11-14: ~2.2-2.5 GiB of /var is already used on each node by +# everything else, so this cache (<=5Gi) and buildkit's (<=6Gi) cannot both fit on one node. The +# third big /var tenant, the act-runner Docker cache, moved off /var the same day +# (apps/wave-1/gitea-actions.yaml, `dind-storage`), so sharing prod-2 with Gitea and the runner no +# longer stacks caches and the prod-2 preference stays. Weight 100 on both: when prod-2 is +# unavailable the scanner prefers prod-1 over buildkit's node. Soft, for the same reason as above. apiVersion: apps/v1 kind: Deployment metadata: @@ -61,6 +70,16 @@ spec: operator: In values: - k3s-prod-2 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + namespaces: + - buildkit + labelSelector: + matchLabels: + app.kubernetes.io/name: buildkitd initContainers: - name: trivy-tools # TRIVY 0.76.1 — pinned by digest (R-VULN-09). Not 0.69.4/5/6 (compromised release, March @@ -148,15 +167,24 @@ spec: - { name: metrics, containerPort: 9090, protocol: TCP } # Sized to trivy, the only heavy thing here: scans are sequential, the limit is the peak of # one large image; the emptyDir sizeLimit bounds disk. + # + # ephemeral-storage, RESIZED 2026-11-14. It was request 256Mi / limit 12Gi with a 10Gi + # cache sizeLimit, on a node /var of ~11.6 GiB: neither bound could fire before kubelet's + # node-level DiskPressure eviction did, which takes other pods down with it. kubelet + # compares the POD's usage (its emptyDirs + container writable layers + logs) against the + # SUM of its containers' limits, and only this container carries one, so this limit must + # cover cache (5Gi) + tools (512Mi) + logs: 6Gi. The request is the measured steady state + # (3.12 GiB of cache: Trivy DB 1.7G + Java DB 1.5G, 2026-11-14) so the scheduler stops + # treating this pod as ~free of local disk. resources: requests: cpu: 100m memory: 256Mi - ephemeral-storage: 256Mi + ephemeral-storage: 3584Mi limits: cpu: "1" memory: 2Gi - ephemeral-storage: 12Gi + ephemeral-storage: 6Gi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -280,6 +308,13 @@ spec: claimName: vuln-scanner-data # Trivy DB + per-image scan cache + TMPDIR (trivy extracts layers here). Size-limited; never # on a replicated volume or in etcd (docs/17 "Load"). A full run cleans it between images. + # + # 5Gi, down from 10Gi (2026-11-14): 10Gi was most of the node's whole ~11.6 GiB /var, so it + # could never fire before node DiskPressure. 5Gi = the 3.12 GiB of DBs measured 2026-11-14 + # plus ~1.9 GiB for one image's extraction (TMPDIR points here, so extraction is inside the + # bound) and a DB refresh. Overrunning it evicts THIS pod only; main.py records the + # in-flight image as failed and sits it out, so an oversized image cannot loop the scanner. + # If the DBs grow past ~4 GiB, raise this AND the scanner's ephemeral-storage limit together. - name: cache emptyDir: - sizeLimit: 10Gi + sizeLimit: 5Gi From 89ed1af282299cacd0d5b2e8d298aed3709646d0 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 14 Nov 2026 14:16:04 -0600 Subject: [PATCH 2/2] docs(decisions): record gate result for bound-var-ephemeral-caches Co-Authored-By: Claude Opus 5.5 --- decisions/2026-11-14-bound-var-ephemeral-caches.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/decisions/2026-11-14-bound-var-ephemeral-caches.md b/decisions/2026-11-14-bound-var-ephemeral-caches.md index f028021..767d4d9 100644 --- a/decisions/2026-11-14-bound-var-ephemeral-caches.md +++ b/decisions/2026-11-14-bound-var-ephemeral-caches.md @@ -65,7 +65,12 @@ evictions). 3584Mi scanner request schedules. - No Kyverno policy selects the non-app `gitea-actions` namespace for volume shape. None of the touched paths are in `infra/gitea/approval-policy.json` `sensitive_paths`. -- Toolbox gate: see the PR description for the exact rc and stages that did not run. +- Toolbox gate from this worktree (DEV_KUBECONFIG=/dev/null, ENGINE shim mounting the main + `.git`, spec repo mounted by dev.sh as a sibling), on the committed HEAD: **rc 3, UNDETERMINED, + not green.** "Stages that did not run: gitea-settings". `conformance` (L2/L3) logged SKIPPED, no + cluster. spec-immutability, approval-policy (N/A offline, proves nothing), decisions-log, + lint, l0 (incl. `flux-postbuild-substitution` and `vuln-scanner-supply-chain` PASS), + render-golden and l1 green. ## What I did NOT verify