Compare commits

..
Author SHA1 Message Date
renovate-bot 7e612212a0 chore(deps): Update docker.io/library/python Docker tag to v3.15.2
renovate/stability-days Updates have not met minimum release age requirement
gate / gate (pull_request) Successful in 46s
gate / conformance (pull_request) Skipped
2026-11-13 08:17:48 +00:00
10 changed files with 13 additions and 266 deletions

No files matched your search

+1 -1
View File
@@ -78,7 +78,7 @@ spec:
spec:
containers:
- name: jackett
image: lscr.io/linuxserver/jackett:v0.24.3053-ls34
image: lscr.io/linuxserver/jackett:v0.24.3030-ls30
ports:
- containerPort: 9117
volumeMounts:
+2 -131
View File
@@ -135,13 +135,6 @@ spec:
# Bump when act-runner-config changes without any other pod-template change, so the
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
# Velero's daily schedule is opt-OUT (defaultVolumesToFsBackup: true) and its volume policy
# skips only emptyDir and NFS (platform/velero/volume-policies.yaml). dind-storage is now a
# Longhorn-backed generic ephemeral volume (see `volumes:`), which that policy does NOT
# match, so without this the nightly backup would push the whole Docker image cache to R2
# and race this pod's replacement (the PartiallyFailed failure that file documents). It is a
# disposable cache: nothing in it is state.
backup.velero.io/backup-volumes-excludes: dind-storage
spec:
# CI holds no cluster credentials — do not mount a SA token into the runner or
# (via the shared netns) the job containers (R-SEC-02, docs/14).
@@ -204,18 +197,12 @@ spec:
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
# ephemeral-storage: /var/lib/docker (images, job containers, their logs) is on the
# dind-storage PVC, NOT the node's /var, so this bounds only dockerd's own rootfs
# writes and its container log. Before 2026-11-14 the image cache was an unbounded
# emptyDir on /var (see `volumes:`).
resources:
requests:
cpu: "2"
memory: 1Gi
ephemeral-storage: 256Mi
limits:
memory: 4Gi
ephemeral-storage: 2Gi
startupProbe:
exec:
command:
@@ -279,132 +266,16 @@ spec:
requests:
cpu: 100m
memory: 256Mi
ephemeral-storage: 128Mi
limits:
memory: 1Gi
ephemeral-storage: 1Gi
# ---- docker-prune: keeps the dind image cache bounded ----------------------------------
#
# dockerd never garbage-collects images. Every new renovate/toolbox digest (~2 GB each) used
# to stack up in /var/lib/docker until the node's /var filled: on 2026-11-12 that tipped
# k3s-prod-2 into DiskPressure and the act-runner pod was evicted 63 times. Moving the cache
# to its own volume (see `volumes:`) stops it hurting the node; this stops it filling that
# volume and failing jobs with ENOSPC instead.
#
# Every 10 minutes: if the volume is at or above 45% used, `docker system prune --all
# --force` removes every image, stopped container, unused network and build cache entry NOT
# in use by a container. Images a running job uses are never removed. 45% (not 85%) so the
# volume stays above the 50%-free gate of PersistentVolumeFillingUpPredicted
# (platform/monitoring/pvc-capacity-alerts.yaml) and a normal refill never pages anyone.
# Threshold-triggered, not age-based: `--filter until=` on images is image CREATION time, so
# an age-based prune would delete the long-lived job images (catthehacker, the toolbox) between
# every pair of jobs and force a re-pull each time.
#
# Accepted race: a prune landing in the milliseconds between a job's image pull and its
# container create removes that image and fails that one job. It needs the volume over the
# threshold AT that instant; a re-run passes.
#
# NO DOLLAR SIGNS, deliberately: clusters/base/apps-wave-1.yaml runs Flux postBuild
# substitution over this whole file, comments included, which rewrites shell variables and
# command substitutions (conformance/tests/l0-static/flux-postbuild-substitution). The
# percentage test is therefore
# a regex over `df -P` output rather than arithmetic on a captured value.
- name: docker-prune
image: docker:29-cli
command:
- /bin/sh
- -c
- |
trap 'exit 0' TERM INT
echo "docker-prune: started; prunes when /var/lib/docker is at or above 45% used"
while true; do
if df -P /var/lib/docker | grep -qE ' (4[5-9]|[5-9][0-9]|100)% '; then
echo "docker-prune: threshold reached, before:"
df -P /var/lib/docker | tail -n 1
docker system prune --all --force || echo "docker-prune: prune failed; retrying next cycle"
echo "docker-prune: after:"
df -P /var/lib/docker | tail -n 1
fi
sleep 600 &
wait
done
env:
- name: DOCKER_HOST
value: tcp://localhost:2376
- name: DOCKER_CERT_PATH
value: /certs/client
- name: DOCKER_TLS_VERIFY
value: "1"
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- name: certs
mountPath: /certs
readOnly: true
# Mounted only so `df` can read the volume's usage; the prune itself goes through the
# daemon's API.
- name: dind-storage
mountPath: /var/lib/docker
readOnly: true
resources:
requests:
cpu: 10m
memory: 32Mi
ephemeral-storage: 16Mi
limits:
memory: 128Mi
ephemeral-storage: 128Mi
volumes:
- name: data
persistentVolumeClaim:
claimName: act-runner-data
- name: certs
emptyDir:
sizeLimit: 16Mi # a handful of PEM files
emptyDir: {}
- name: config
configMap:
name: act-runner-config
# The Docker daemon's data root: every job image and job container. OFF the node's /var.
#
# WHY NOT an emptyDir (what this was until 2026-11-14). kubelet emptyDirs live on /var, a
# separate 11.8 GiB partition on every node, and containerd's image store is NOT there
# (k3s --data-dir is /data/k3s, infra/ansible/inventory/group_vars/k3s_prod/vars.yml), so
# kubelet image GC never frees /var. Measured 2026-11-14: this emptyDir held 3.75 GiB on
# k3s-prod-2 (renovate 1.99 GB + toolbox 2.04 GB) with ~1.9 GiB of /var left; the
# 2026-11-12 eviction storm was it tipping the node into DiskPressure. A sizeLimit alone
# would only trade that for this pod evicting itself, and /var is too small to give the
# cache the room it needs next to vuln-scanner's and buildkit's.
#
# WHY a GENERIC EPHEMERAL volume on `longhorn-scratch` and not a standalone PVC.
# * Off /var: Longhorn's dataPath is /data/longhorn on the ~424 GB partition (~170 GB free
# per node, measured 2026-11-14).
# * Single replica (`longhorn-scratch`, dataLocality best-effort): a disposable cache gains
# nothing from 3x replication, and replica writes of multi-GB image pulls would land on
# the device etcd's WAL shares (docs/04 accepted risk). Same reasoning as CDI's scratch,
# which is what that class exists for.
# * Pod-lifetime, not persistent: the claim is created with the pod and deleted with it.
# That keeps this out of R-STATE-02 ("persistent storage MUST be replicated"), which a
# long-lived single-replica PVC would violate. It also means a replacement pod never
# waits on a lone replica stranded on a dead node: it gets a fresh, empty volume. The
# cost, accepted: the cache is cold after every pod replacement (~4 GB of pulls), which
# is exactly what the emptyDir already did.
# * Bounded twice: the claim size is a hard ceiling (ENOSPC for jobs, never a node
# eviction), and the docker-prune container keeps it under 45%.
# Velero: excluded by the pod annotation above AND the claim label below.
- name: dind-storage
ephemeral:
volumeClaimTemplate:
metadata:
labels:
app.kubernetes.io/name: act-runner
velero.io/exclude-from-backup: "true"
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn-scratch
resources:
requests:
storage: 30Gi
emptyDir: {}
+1 -1
View File
@@ -78,7 +78,7 @@ spec:
spec:
containers:
- name: semaphore
image: semaphoreui/semaphore:v2.20.24
image: semaphoreui/semaphore:v2.20.23
ports:
- containerPort: 3000
env:
+1 -1
View File
@@ -75,7 +75,7 @@ spec:
spec:
containers:
- name: seerr
image: ghcr.io/seerr-team/seerr:v3.5.2
image: ghcr.io/seerr-team/seerr:v3.5.1
ports:
- containerPort: 5055
volumeMounts:
+1 -1
View File
@@ -16,7 +16,7 @@
name: jackett
kind: container
source:
image: lscr.io/linuxserver/jackett:v0.24.3053-ls34
image: lscr.io/linuxserver/jackett:v0.24.3030-ls30
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
storage:
+1 -1
View File
@@ -24,7 +24,7 @@
name: seerr
kind: container
source:
image: ghcr.io/seerr-team/seerr:v3.5.2 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
image: ghcr.io/seerr-team/seerr:v3.5.1 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
# The ghcr.io/seerr-team/seerr image runs as the non-root user `node:node` (UID 1000 / GID 1000 —
# the standard node base-image user; confirmed from the image config's User field). On boot the app
# creates /app/config/logs and writes its settings/DB under the mounted /app/config PVC. A fresh
+1 -1
View File
@@ -22,7 +22,7 @@
name: semaphore
kind: container
source:
image: semaphoreui/semaphore:v2.20.24 # pinned recent stable tag (R-UPD-01); update.range governs advancement
image: semaphoreui/semaphore:v2.20.23 # pinned recent stable tag (R-UPD-01); update.range governs advancement
env:
# --- PostgreSQL connection (the CNPG-DB pattern). DIALECT/HOST/PORT/DB/USER are literals; PASS is
# sourced from the in-namespace SOPS secret (the CNPG -app secret lives in ns semaphore-db,
@@ -1,89 +0,0 @@
---
date: 2026-11-14
branch: fix/bound-var-ephemeral-caches
title: Bound the three caches filling each node's /var (act-runner off /var, vuln-scanner fitted)
---
## What changed
Each node's `/var` is a separate ~11.6 GiB partition holding kubelet emptyDirs and pod logs;
containerd's image store is on `/data/k3s`, so kubelet image GC never frees `/var`. On 2026-11-12
the act-runner's unbounded Docker cache tipped k3s-prod-2 into DiskPressure (63 act-runner
evictions).
- `apps/wave-1/gitea-actions.yaml` (hand-authored: a renderer `EXTRA_UNITS` file, placed by
`catalog/render.py` but never generated, so no re-render was needed or run):
- `dind-storage` is now a **generic ephemeral volume** (30Gi, `longhorn-scratch`) instead of an
unbounded `emptyDir {}`. Off `/var`, onto `/data/longhorn`.
- New `docker-prune` container (`docker:29-cli`): every 10 min, if the volume is >=45% used,
`docker system prune --all --force`. Threshold-based, because image `until=` filters use image
creation time and would re-pull long-lived job images between jobs. No `$` in the script
(Flux postBuild substitution runs over the file).
- ephemeral-storage requests/limits on dind (256Mi/2Gi), runner (128Mi/1Gi) and prune
(16Mi/128Mi); `certs` emptyDir `sizeLimit: 16Mi`.
- Velero opt-out: pod annotation `backup.velero.io/backup-volumes-excludes: dind-storage` and
the claim label `velero.io/exclude-from-backup: "true"`. Daily schedule is opt-out and its
volume policy skips only emptyDir/NFS, so without this the cache would go to R2 nightly.
- `platform/vuln-scanner/deployment.yaml`: `/cache` emptyDir sizeLimit 10Gi -> 5Gi; scanner
ephemeral-storage request 256Mi -> 3584Mi (measured steady state), limit 12Gi -> 6Gi (covers
cache + tools + logs, since kubelet compares pod usage with the sum of container limits). Added
soft pod anti-affinity (weight 100) from `buildkit/buildkitd`. The prod-2 node preference is kept:
with the runner off `/var`, sharing prod-2 no longer stacks caches.
- buildkit: unchanged (maxUsedSpace 4GB, sizeLimit 6Gi, bounded). It has no anti-affinity back to
the scanner; the scanner side alone carries it.
### Why not what the brief preferred
- **Persistent PVC for the runner cache** (survives pod moves): not done. A long-lived
single-replica PVC is persistent storage that is not replicated, against R-STATE-02. A 3-replica
PVC would triple multi-GB pull writes onto the device etcd's WAL shares. A generic ephemeral
volume is pod-lifetime, the same class as the emptyDir it replaces. It is also never stranded on a
dead node: a replacement pod gets a fresh claim. Cost: the cache is cold after a pod replacement,
which is what the emptyDir already did.
- **vuln-scanner cache onto a PVC**: not done. docs/17 "Load" says scan caches MUST NOT be on a
replicated Longhorn volume *because etcd and Longhorn share a partition*. A single-replica volume
meets the letter but not the reason. L0 `vuln-scanner-supply-chain` also requires the cache to be
a size-limited emptyDir.
### Spec escalations (for the architect, not acted on)
1. docs/04 calls CDI scratch "the one non-replicated use of Longhorn". `longhorn-scratch` now also
backs the act-runner's pod-lifetime Docker cache. The sentence should say "throwaway,
pod-lifetime scratch", or name this use.
2. docs/04's rebuild rule and R-STATE-02 don't say whether a generic ephemeral volume counts as
"persistent storage". This change reads it as not.
## What I verified
- Live, read-only, 2026-11-14 (kubelet `stats/summary`): nodefs capacity/available prod-1
11.60/8.89 GiB, prod-2 11.85/1.93, prod-3 11.60/4.75. prod-2 `dind-storage` 3.75 GiB, vuln
`cache` 3.12 GiB; prod-3 buildkit `cache` 3.86 GiB. Non-cache `/var` use ~2.1-2.5 GiB per node.
- StorageClasses live: `longhorn-scratch` = `numberOfReplicas: "1"`, `dataLocality: best-effort`,
reclaim Delete. Longhorn disk available per node 177-187 GB; over-provisioning 200%, minimal
available 25%.
- Node allocatable ephemeral-storage 11.8-12.1 GB; current requests prod-2 256Mi, prod-3 2Gi, so the
3584Mi scanner request schedules.
- No Kyverno policy selects the non-app `gitea-actions` namespace for volume shape. None of the
touched paths are in `infra/gitea/approval-policy.json` `sensitive_paths`.
- Toolbox gate from this worktree (DEV_KUBECONFIG=/dev/null, ENGINE shim mounting the main
`.git`, spec repo mounted by dev.sh as a sibling), on the committed HEAD: **rc 3, UNDETERMINED,
not green.** "Stages that did not run: gitea-settings". `conformance` (L2/L3) logged SKIPPED, no
cluster. spec-immutability, approval-policy (N/A offline, proves nothing), decisions-log,
lint, l0 (incl. `flux-postbuild-substitution` and `vuln-scanner-supply-chain` PASS),
render-golden and l1 green.
## What I did NOT verify
Nothing here has been deployed. This is only provable after merge, under Flux.
- That the generic ephemeral claim provisions and attaches on `longhorn-scratch`, and that dind
starts on it within apps-wave-1's 5m health timeout (`wait: true`). If it doesn't, wave 1 stalls,
and with it every wave above it.
- That `docker system prune` runs: TLS to dind from the sidecar, BusyBox `df -P` output matching
the regex, cert file permissions readable by a capability-less root process. UNVERIFIED.
- That Velero skips the volume and the claim. The annotation and label are documented Velero
opt-outs but were not exercised here.
- That 5Gi is enough for the largest image Trivy scans. Peak extraction size was not measured. An
overrun evicts only the scanner, and the image sits out per the in-flight marker.
- `runbooks/node-disk.md` still points at `/var/lib/rancher/k3s` for containerd. That is stale
(it's on `/data/k3s`) and not fixed here.
+1 -1
View File
@@ -3205,7 +3205,7 @@ spec:
# sibling's pod on prod runs a different digest under this same tag.
# Renovate's `kubernetes` manager covers platform/** (renovate.json5), so
# it proposes tag and digest bumps for this line like any other image.
image: python:3.15.2-alpine3.23@sha256:32f06114a111c2467804d06194c5584f16804a78cfaf427fb1fcbb0b3391d0f1
image: python:3.15.2-alpine3.23@sha256:8d77d813a53e534dec9e1e5182cbd5b651b70d63b27e4ea7c1f616b37cf9aad2
imagePullPolicy: IfNotPresent
command:
- python3
+4 -39
View File
@@ -14,15 +14,6 @@
#
# SOFT affinity toward k3s-prod-2: prod-1 is the node whose etcd I/O starved historically, and scans
# are CPU- and I/O-heavy. Soft on purpose: a scanner that cannot be placed must still run elsewhere.
#
# SOFT anti-affinity from buildkitd (added 2026-11-14). The /cache emptyDir lives on the node's /var,
# a separate ~11.6 GiB partition kubelet evicts from at <5% free, and buildkitd's own /var cache is
# bounded at 6Gi. Measured 2026-11-14: ~2.2-2.5 GiB of /var is already used on each node by
# everything else, so this cache (<=5Gi) and buildkit's (<=6Gi) cannot both fit on one node. The
# third big /var tenant, the act-runner Docker cache, moved off /var the same day
# (apps/wave-1/gitea-actions.yaml, `dind-storage`), so sharing prod-2 with Gitea and the runner no
# longer stacks caches and the prod-2 preference stays. Weight 100 on both: when prod-2 is
# unavailable the scanner prefers prod-1 over buildkit's node. Soft, for the same reason as above.
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -70,16 +61,6 @@ spec:
operator: In
values:
- k3s-prod-2
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
topologyKey: kubernetes.io/hostname
namespaces:
- buildkit
labelSelector:
matchLabels:
app.kubernetes.io/name: buildkitd
initContainers:
- name: trivy-tools
# TRIVY 0.76.1 — pinned by digest (R-VULN-09). Not 0.69.4/5/6 (compromised release, March
@@ -128,7 +109,7 @@ spec:
# (19 days old; 3.14.10-alpine, the current 3.14-alpine, was built 2026-11-05 and is under
# the 7-day age applied here by hand). Tracked by the `kubernetes` Renovate manager.
# Runs as 65532 from this pod's securityContext; the image's default is root, overridden.
image: docker.io/library/python:3.14.9-alpine@sha256:dbeb11d16bb95c3a7e3c25c842cb9f20cbc8c18f8be0461a4a0dcc861ad386a0
image: docker.io/library/python:3.15.2-alpine@sha256:0f9552bb28711333da16915ec146f5e6bb44a7fdac9ced15dc17c2e0206346dd
imagePullPolicy: IfNotPresent
command:
- python3
@@ -167,24 +148,15 @@ spec:
- { name: metrics, containerPort: 9090, protocol: TCP }
# Sized to trivy, the only heavy thing here: scans are sequential, the limit is the peak of
# one large image; the emptyDir sizeLimit bounds disk.
#
# ephemeral-storage, RESIZED 2026-11-14. It was request 256Mi / limit 12Gi with a 10Gi
# cache sizeLimit, on a node /var of ~11.6 GiB: neither bound could fire before kubelet's
# node-level DiskPressure eviction did, which takes other pods down with it. kubelet
# compares the POD's usage (its emptyDirs + container writable layers + logs) against the
# SUM of its containers' limits, and only this container carries one, so this limit must
# cover cache (5Gi) + tools (512Mi) + logs: 6Gi. The request is the measured steady state
# (3.12 GiB of cache: Trivy DB 1.7G + Java DB 1.5G, 2026-11-14) so the scheduler stops
# treating this pod as ~free of local disk.
resources:
requests:
cpu: 100m
memory: 256Mi
ephemeral-storage: 3584Mi
ephemeral-storage: 256Mi
limits:
cpu: "1"
memory: 2Gi
ephemeral-storage: 6Gi
ephemeral-storage: 12Gi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
@@ -308,13 +280,6 @@ spec:
claimName: vuln-scanner-data
# Trivy DB + per-image scan cache + TMPDIR (trivy extracts layers here). Size-limited; never
# on a replicated volume or in etcd (docs/17 "Load"). A full run cleans it between images.
#
# 5Gi, down from 10Gi (2026-11-14): 10Gi was most of the node's whole ~11.6 GiB /var, so it
# could never fire before node DiskPressure. 5Gi = the 3.12 GiB of DBs measured 2026-11-14
# plus ~1.9 GiB for one image's extraction (TMPDIR points here, so extraction is inside the
# bound) and a DB refresh. Overrunning it evicts THIS pod only; main.py records the
# in-flight image as failed and sits it out, so an oversized image cannot loop the scanner.
# If the DBs grow past ~4 GiB, raise this AND the scanner's ephemeral-storage limit together.
- name: cache
emptyDir:
sizeLimit: 5Gi
sizeLimit: 10Gi