Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  jakman85andClaude Opus 5.5
|
89ed1af282 | ||
|
jakman85andClaude Opus 5.5
|
3ae7fb74f8 |
No files matched your search
@@ -135,6 +135,13 @@ spec:
|
||||
# Bump when act-runner-config changes without any other pod-template change, so the
|
||||
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
|
||||
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
|
||||
# Velero's daily schedule is opt-OUT (defaultVolumesToFsBackup: true) and its volume policy
|
||||
# skips only emptyDir and NFS (platform/velero/volume-policies.yaml). dind-storage is now a
|
||||
# Longhorn-backed generic ephemeral volume (see `volumes:`), which that policy does NOT
|
||||
# match, so without this the nightly backup would push the whole Docker image cache to R2
|
||||
# and race this pod's replacement (the PartiallyFailed failure that file documents). It is a
|
||||
# disposable cache: nothing in it is state.
|
||||
backup.velero.io/backup-volumes-excludes: dind-storage
|
||||
spec:
|
||||
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
||||
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
||||
@@ -197,12 +204,18 @@ spec:
|
||||
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
|
||||
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
|
||||
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
|
||||
# ephemeral-storage: /var/lib/docker (images, job containers, their logs) is on the
|
||||
# dind-storage PVC, NOT the node's /var, so this bounds only dockerd's own rootfs
|
||||
# writes and its container log. Before 2026-11-14 the image cache was an unbounded
|
||||
# emptyDir on /var (see `volumes:`).
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 256Mi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
ephemeral-storage: 2Gi
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
@@ -266,16 +279,132 @@ spec:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
ephemeral-storage: 128Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 1Gi
|
||||
# ---- docker-prune: keeps the dind image cache bounded ----------------------------------
|
||||
#
|
||||
# dockerd never garbage-collects images. Every new renovate/toolbox digest (~2 GB each) used
|
||||
# to stack up in /var/lib/docker until the node's /var filled: on 2026-11-12 that tipped
|
||||
# k3s-prod-2 into DiskPressure and the act-runner pod was evicted 63 times. Moving the cache
|
||||
# to its own volume (see `volumes:`) stops it hurting the node; this stops it filling that
|
||||
# volume and failing jobs with ENOSPC instead.
|
||||
#
|
||||
# Every 10 minutes: if the volume is at or above 45% used, `docker system prune --all
|
||||
# --force` removes every image, stopped container, unused network and build cache entry NOT
|
||||
# in use by a container. Images a running job uses are never removed. 45% (not 85%) so the
|
||||
# volume stays above the 50%-free gate of PersistentVolumeFillingUpPredicted
|
||||
# (platform/monitoring/pvc-capacity-alerts.yaml) and a normal refill never pages anyone.
|
||||
# Threshold-triggered, not age-based: `--filter until=` on images is image CREATION time, so
|
||||
# an age-based prune would delete the long-lived job images (catthehacker, the toolbox) between
|
||||
# every pair of jobs and force a re-pull each time.
|
||||
#
|
||||
# Accepted race: a prune landing in the milliseconds between a job's image pull and its
|
||||
# container create removes that image and fails that one job. It needs the volume over the
|
||||
# threshold AT that instant; a re-run passes.
|
||||
#
|
||||
# NO DOLLAR SIGNS, deliberately: clusters/base/apps-wave-1.yaml runs Flux postBuild
|
||||
# substitution over this whole file, comments included, which rewrites shell variables and
|
||||
# command substitutions (conformance/tests/l0-static/flux-postbuild-substitution). The
|
||||
# percentage test is therefore
|
||||
# a regex over `df -P` output rather than arithmetic on a captured value.
|
||||
- name: docker-prune
|
||||
image: docker:29-cli
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
trap 'exit 0' TERM INT
|
||||
echo "docker-prune: started; prunes when /var/lib/docker is at or above 45% used"
|
||||
while true; do
|
||||
if df -P /var/lib/docker | grep -qE ' (4[5-9]|[5-9][0-9]|100)% '; then
|
||||
echo "docker-prune: threshold reached, before:"
|
||||
df -P /var/lib/docker | tail -n 1
|
||||
docker system prune --all --force || echo "docker-prune: prune failed; retrying next cycle"
|
||||
echo "docker-prune: after:"
|
||||
df -P /var/lib/docker | tail -n 1
|
||||
fi
|
||||
sleep 600 &
|
||||
wait
|
||||
done
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://localhost:2376
|
||||
- name: DOCKER_CERT_PATH
|
||||
value: /certs/client
|
||||
- name: DOCKER_TLS_VERIFY
|
||||
value: "1"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: certs
|
||||
mountPath: /certs
|
||||
readOnly: true
|
||||
# Mounted only so `df` can read the volume's usage; the prune itself goes through the
|
||||
# daemon's API.
|
||||
- name: dind-storage
|
||||
mountPath: /var/lib/docker
|
||||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
ephemeral-storage: 16Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
ephemeral-storage: 128Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: act-runner-data
|
||||
- name: certs
|
||||
emptyDir: {}
|
||||
emptyDir:
|
||||
sizeLimit: 16Mi # a handful of PEM files
|
||||
- name: config
|
||||
configMap:
|
||||
name: act-runner-config
|
||||
# The Docker daemon's data root: every job image and job container. OFF the node's /var.
|
||||
#
|
||||
# WHY NOT an emptyDir (what this was until 2026-11-14). kubelet emptyDirs live on /var, a
|
||||
# separate 11.8 GiB partition on every node, and containerd's image store is NOT there
|
||||
# (k3s --data-dir is /data/k3s, infra/ansible/inventory/group_vars/k3s_prod/vars.yml), so
|
||||
# kubelet image GC never frees /var. Measured 2026-11-14: this emptyDir held 3.75 GiB on
|
||||
# k3s-prod-2 (renovate 1.99 GB + toolbox 2.04 GB) with ~1.9 GiB of /var left; the
|
||||
# 2026-11-12 eviction storm was it tipping the node into DiskPressure. A sizeLimit alone
|
||||
# would only trade that for this pod evicting itself, and /var is too small to give the
|
||||
# cache the room it needs next to vuln-scanner's and buildkit's.
|
||||
#
|
||||
# WHY a GENERIC EPHEMERAL volume on `longhorn-scratch` and not a standalone PVC.
|
||||
# * Off /var: Longhorn's dataPath is /data/longhorn on the ~424 GB partition (~170 GB free
|
||||
# per node, measured 2026-11-14).
|
||||
# * Single replica (`longhorn-scratch`, dataLocality best-effort): a disposable cache gains
|
||||
# nothing from 3x replication, and replica writes of multi-GB image pulls would land on
|
||||
# the device etcd's WAL shares (docs/04 accepted risk). Same reasoning as CDI's scratch,
|
||||
# which is what that class exists for.
|
||||
# * Pod-lifetime, not persistent: the claim is created with the pod and deleted with it.
|
||||
# That keeps this out of R-STATE-02 ("persistent storage MUST be replicated"), which a
|
||||
# long-lived single-replica PVC would violate. It also means a replacement pod never
|
||||
# waits on a lone replica stranded on a dead node: it gets a fresh, empty volume. The
|
||||
# cost, accepted: the cache is cold after every pod replacement (~4 GB of pulls), which
|
||||
# is exactly what the emptyDir already did.
|
||||
# * Bounded twice: the claim size is a hard ceiling (ENOSPC for jobs, never a node
|
||||
# eviction), and the docker-prune container keeps it under 45%.
|
||||
# Velero: excluded by the pod annotation above AND the claim label below.
|
||||
- name: dind-storage
|
||||
emptyDir: {}
|
||||
ephemeral:
|
||||
volumeClaimTemplate:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: act-runner
|
||||
velero.io/exclude-from-backup: "true"
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn-scratch
|
||||
resources:
|
||||
requests:
|
||||
storage: 30Gi
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
date: 2026-11-14
|
||||
branch: fix/bound-var-ephemeral-caches
|
||||
title: Bound the three caches filling each node's /var (act-runner off /var, vuln-scanner fitted)
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
Each node's `/var` is a separate ~11.6 GiB partition holding kubelet emptyDirs and pod logs;
|
||||
containerd's image store is on `/data/k3s`, so kubelet image GC never frees `/var`. On 2026-11-12
|
||||
the act-runner's unbounded Docker cache tipped k3s-prod-2 into DiskPressure (63 act-runner
|
||||
evictions).
|
||||
|
||||
- `apps/wave-1/gitea-actions.yaml` (hand-authored: a renderer `EXTRA_UNITS` file, placed by
|
||||
`catalog/render.py` but never generated, so no re-render was needed or run):
|
||||
- `dind-storage` is now a **generic ephemeral volume** (30Gi, `longhorn-scratch`) instead of an
|
||||
unbounded `emptyDir {}`. Off `/var`, onto `/data/longhorn`.
|
||||
- New `docker-prune` container (`docker:29-cli`): every 10 min, if the volume is >=45% used,
|
||||
`docker system prune --all --force`. Threshold-based, because image `until=` filters use image
|
||||
creation time and would re-pull long-lived job images between jobs. No `$` in the script
|
||||
(Flux postBuild substitution runs over the file).
|
||||
- ephemeral-storage requests/limits on dind (256Mi/2Gi), runner (128Mi/1Gi) and prune
|
||||
(16Mi/128Mi); `certs` emptyDir `sizeLimit: 16Mi`.
|
||||
- Velero opt-out: pod annotation `backup.velero.io/backup-volumes-excludes: dind-storage` and
|
||||
the claim label `velero.io/exclude-from-backup: "true"`. Daily schedule is opt-out and its
|
||||
volume policy skips only emptyDir/NFS, so without this the cache would go to R2 nightly.
|
||||
- `platform/vuln-scanner/deployment.yaml`: `/cache` emptyDir sizeLimit 10Gi -> 5Gi; scanner
|
||||
ephemeral-storage request 256Mi -> 3584Mi (measured steady state), limit 12Gi -> 6Gi (covers
|
||||
cache + tools + logs, since kubelet compares pod usage with the sum of container limits). Added
|
||||
soft pod anti-affinity (weight 100) from `buildkit/buildkitd`. The prod-2 node preference is kept:
|
||||
with the runner off `/var`, sharing prod-2 no longer stacks caches.
|
||||
- buildkit: unchanged (maxUsedSpace 4GB, sizeLimit 6Gi, bounded). It has no anti-affinity back to
|
||||
the scanner; the scanner side alone carries it.
|
||||
|
||||
### Why not what the brief preferred
|
||||
|
||||
- **Persistent PVC for the runner cache** (survives pod moves): not done. A long-lived
|
||||
single-replica PVC is persistent storage that is not replicated, against R-STATE-02. A 3-replica
|
||||
PVC would triple multi-GB pull writes onto the device etcd's WAL shares. A generic ephemeral
|
||||
volume is pod-lifetime, the same class as the emptyDir it replaces. It is also never stranded on a
|
||||
dead node: a replacement pod gets a fresh claim. Cost: the cache is cold after a pod replacement,
|
||||
which is what the emptyDir already did.
|
||||
- **vuln-scanner cache onto a PVC**: not done. docs/17 "Load" says scan caches MUST NOT be on a
|
||||
replicated Longhorn volume *because etcd and Longhorn share a partition*. A single-replica volume
|
||||
meets the letter but not the reason. L0 `vuln-scanner-supply-chain` also requires the cache to be
|
||||
a size-limited emptyDir.
|
||||
|
||||
### Spec escalations (for the architect, not acted on)
|
||||
|
||||
1. docs/04 calls CDI scratch "the one non-replicated use of Longhorn". `longhorn-scratch` now also
|
||||
backs the act-runner's pod-lifetime Docker cache. The sentence should say "throwaway,
|
||||
pod-lifetime scratch", or name this use.
|
||||
2. docs/04's rebuild rule and R-STATE-02 don't say whether a generic ephemeral volume counts as
|
||||
"persistent storage". This change reads it as not.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Live, read-only, 2026-11-14 (kubelet `stats/summary`): nodefs capacity/available prod-1
|
||||
11.60/8.89 GiB, prod-2 11.85/1.93, prod-3 11.60/4.75. prod-2 `dind-storage` 3.75 GiB, vuln
|
||||
`cache` 3.12 GiB; prod-3 buildkit `cache` 3.86 GiB. Non-cache `/var` use ~2.1-2.5 GiB per node.
|
||||
- StorageClasses live: `longhorn-scratch` = `numberOfReplicas: "1"`, `dataLocality: best-effort`,
|
||||
reclaim Delete. Longhorn disk available per node 177-187 GB; over-provisioning 200%, minimal
|
||||
available 25%.
|
||||
- Node allocatable ephemeral-storage 11.8-12.1 GB; current requests prod-2 256Mi, prod-3 2Gi, so the
|
||||
3584Mi scanner request schedules.
|
||||
- No Kyverno policy selects the non-app `gitea-actions` namespace for volume shape. None of the
|
||||
touched paths are in `infra/gitea/approval-policy.json` `sensitive_paths`.
|
||||
- Toolbox gate from this worktree (DEV_KUBECONFIG=/dev/null, ENGINE shim mounting the main
|
||||
`.git`, spec repo mounted by dev.sh as a sibling), on the committed HEAD: **rc 3, UNDETERMINED,
|
||||
not green.** "Stages that did not run: gitea-settings". `conformance` (L2/L3) logged SKIPPED, no
|
||||
cluster. spec-immutability, approval-policy (N/A offline, proves nothing), decisions-log,
|
||||
lint, l0 (incl. `flux-postbuild-substitution` and `vuln-scanner-supply-chain` PASS),
|
||||
render-golden and l1 green.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
Nothing here has been deployed. This is only provable after merge, under Flux.
|
||||
|
||||
- That the generic ephemeral claim provisions and attaches on `longhorn-scratch`, and that dind
|
||||
starts on it within apps-wave-1's 5m health timeout (`wait: true`). If it doesn't, wave 1 stalls,
|
||||
and with it every wave above it.
|
||||
- That `docker system prune` runs: TLS to dind from the sidecar, BusyBox `df -P` output matching
|
||||
the regex, cert file permissions readable by a capability-less root process. UNVERIFIED.
|
||||
- That Velero skips the volume and the claim. The annotation and label are documented Velero
|
||||
opt-outs but were not exercised here.
|
||||
- That 5Gi is enough for the largest image Trivy scans. Peak extraction size was not measured. An
|
||||
overrun evicts only the scanner, and the image sits out per the in-flight marker.
|
||||
- `runbooks/node-disk.md` still points at `/var/lib/rancher/k3s` for containerd. That is stale
|
||||
(it's on `/data/k3s`) and not fixed here.
|
||||
@@ -14,6 +14,15 @@
|
||||
#
|
||||
# SOFT affinity toward k3s-prod-2: prod-1 is the node whose etcd I/O starved historically, and scans
|
||||
# are CPU- and I/O-heavy. Soft on purpose: a scanner that cannot be placed must still run elsewhere.
|
||||
#
|
||||
# SOFT anti-affinity from buildkitd (added 2026-11-14). The /cache emptyDir lives on the node's /var,
|
||||
# a separate ~11.6 GiB partition kubelet evicts from at <5% free, and buildkitd's own /var cache is
|
||||
# bounded at 6Gi. Measured 2026-11-14: ~2.2-2.5 GiB of /var is already used on each node by
|
||||
# everything else, so this cache (<=5Gi) and buildkit's (<=6Gi) cannot both fit on one node. The
|
||||
# third big /var tenant, the act-runner Docker cache, moved off /var the same day
|
||||
# (apps/wave-1/gitea-actions.yaml, `dind-storage`), so sharing prod-2 with Gitea and the runner no
|
||||
# longer stacks caches and the prod-2 preference stays. Weight 100 on both: when prod-2 is
|
||||
# unavailable the scanner prefers prod-1 over buildkit's node. Soft, for the same reason as above.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -61,6 +70,16 @@ spec:
|
||||
operator: In
|
||||
values:
|
||||
- k3s-prod-2
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
namespaces:
|
||||
- buildkit
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: buildkitd
|
||||
initContainers:
|
||||
- name: trivy-tools
|
||||
# TRIVY 0.76.1 — pinned by digest (R-VULN-09). Not 0.69.4/5/6 (compromised release, March
|
||||
@@ -148,15 +167,24 @@ spec:
|
||||
- { name: metrics, containerPort: 9090, protocol: TCP }
|
||||
# Sized to trivy, the only heavy thing here: scans are sequential, the limit is the peak of
|
||||
# one large image; the emptyDir sizeLimit bounds disk.
|
||||
#
|
||||
# ephemeral-storage, RESIZED 2026-11-14. It was request 256Mi / limit 12Gi with a 10Gi
|
||||
# cache sizeLimit, on a node /var of ~11.6 GiB: neither bound could fire before kubelet's
|
||||
# node-level DiskPressure eviction did, which takes other pods down with it. kubelet
|
||||
# compares the POD's usage (its emptyDirs + container writable layers + logs) against the
|
||||
# SUM of its containers' limits, and only this container carries one, so this limit must
|
||||
# cover cache (5Gi) + tools (512Mi) + logs: 6Gi. The request is the measured steady state
|
||||
# (3.12 GiB of cache: Trivy DB 1.7G + Java DB 1.5G, 2026-11-14) so the scheduler stops
|
||||
# treating this pod as ~free of local disk.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
ephemeral-storage: 256Mi
|
||||
ephemeral-storage: 3584Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 2Gi
|
||||
ephemeral-storage: 12Gi
|
||||
ephemeral-storage: 6Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
@@ -280,6 +308,13 @@ spec:
|
||||
claimName: vuln-scanner-data
|
||||
# Trivy DB + per-image scan cache + TMPDIR (trivy extracts layers here). Size-limited; never
|
||||
# on a replicated volume or in etcd (docs/17 "Load"). A full run cleans it between images.
|
||||
#
|
||||
# 5Gi, down from 10Gi (2026-11-14): 10Gi was most of the node's whole ~11.6 GiB /var, so it
|
||||
# could never fire before node DiskPressure. 5Gi = the 3.12 GiB of DBs measured 2026-11-14
|
||||
# plus ~1.9 GiB for one image's extraction (TMPDIR points here, so extraction is inside the
|
||||
# bound) and a DB refresh. Overrunning it evicts THIS pod only; main.py records the
|
||||
# in-flight image as failed and sits it out, so an oversized image cannot loop the scanner.
|
||||
# If the DBs grow past ~4 GiB, raise this AND the scanner's ephemeral-storage limit together.
|
||||
- name: cache
|
||||
emptyDir:
|
||||
sizeLimit: 10Gi
|
||||
sizeLimit: 5Gi
|
||||
Reference in new issue
Block a user