Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6150b8957 | ||
|
|
466f63ec3c | ||
|
|
664ee82b9a | ||
|
|
95e08da2f0 | ||
|
|
0d3e51acc4 | ||
|
|
11267409d7 | ||
|
|
6a192ede2a | ||
|
|
06d4a9a1fc | ||
|
|
9db20e8ca0 | ||
|
|
ddd04d460e | ||
|
|
f0760d1024 | ||
|
|
553d027f7c | ||
|
|
183bca6897 | ||
|
|
ce849032d5 | ||
|
|
b766130b85 | ||
|
|
862773f31b | ||
|
|
db4ba79a1c | ||
|
|
0b6282aba4 | ||
|
|
c6cfaf62b6 | ||
|
|
4ae648f6d3 | ||
|
|
a4d5206294 | ||
|
|
47eb6afa02 | ||
|
|
74ba90c0b3 | ||
|
|
fb45d7441b | ||
|
|
8ed4e30201 | ||
|
|
24db269ea8 | ||
|
|
df96be98bf | ||
|
|
b4ff2879be | ||
|
|
d9571fc6d8 | ||
|
|
593d0fad15 | ||
|
|
0c6ee4921d | ||
|
|
d550a501e2 | ||
|
|
7f0cbc7ba2 | ||
|
|
a7eadd4054 |
No files matched your search
@@ -80,7 +80,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0 # full history so ci/touched-requirements.sh can diff vs main
|
fetch-depth: 0 # full history so ci/touched-requirements.sh can diff vs main
|
||||||
|
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ jobs:
|
|||||||
timeout-minutes: 75
|
timeout-minutes: 75
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0 # the per-image change detection diffs against the base commit
|
fetch-depth: 0 # the per-image change detection diffs against the base commit
|
||||||
|
|
||||||
|
|||||||
@@ -60,9 +60,9 @@ jobs:
|
|||||||
spec-immutability:
|
spec-immutability:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4 # the implementation repo
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 # the implementation repo
|
||||||
- name: Check out the read-only spec (sibling path)
|
- name: Check out the read-only spec (sibling path)
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with:
|
with:
|
||||||
repository: ${{ vars.SPEC_REPO_SLUG }} # e.g. Home-Network/home-network-k3s (set in Gitea)
|
repository: ${{ vars.SPEC_REPO_SLUG }} # e.g. Home-Network/home-network-k3s (set in Gitea)
|
||||||
path: ../home-network-k3s
|
path: ../home-network-k3s
|
||||||
@@ -74,7 +74,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [spec-immutability]
|
needs: [spec-immutability]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with: { fetch-depth: 0 } # full history so touched-requirement diff works
|
with: { fetch-depth: 0 } # full history so touched-requirement diff works
|
||||||
- name: Install pinned tools
|
- name: Install pinned tools
|
||||||
run: ./.gitea/workflows/scripts/install-tools.sh lint
|
run: ./.gitea/workflows/scripts/install-tools.sh lint
|
||||||
@@ -85,7 +85,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [lint]
|
needs: [lint]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
- name: Install pinned tools
|
- name: Install pinned tools
|
||||||
run: ./.gitea/workflows/scripts/install-tools.sh l0
|
run: ./.gitea/workflows/scripts/install-tools.sh l0
|
||||||
@@ -96,7 +96,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [l0]
|
needs: [l0]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
- name: Install pinned tools
|
- name: Install pinned tools
|
||||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||||
@@ -107,7 +107,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [render-golden]
|
needs: [render-golden]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
- name: Install pinned tools
|
- name: Install pinned tools
|
||||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||||
@@ -115,7 +115,7 @@ jobs:
|
|||||||
run: make -C ci l1
|
run: make -C ci l1
|
||||||
- name: Upload L1 artifacts
|
- name: Upload L1 artifacts
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||||
with:
|
with:
|
||||||
name: l1-junit-coverage
|
name: l1-junit-coverage
|
||||||
path: |
|
path: |
|
||||||
@@ -131,7 +131,7 @@ jobs:
|
|||||||
needs: [l1]
|
needs: [l1]
|
||||||
if: ${{ vars.CLUSTER_AVAILABLE == 'true' }}
|
if: ${{ vars.CLUSTER_AVAILABLE == 'true' }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
with: { fetch-depth: 0 }
|
with: { fetch-depth: 0 }
|
||||||
- name: Install pinned tools
|
- name: Install pinned tools
|
||||||
run: ./.gitea/workflows/scripts/install-tools.sh conformance
|
run: ./.gitea/workflows/scripts/install-tools.sh conformance
|
||||||
@@ -142,7 +142,7 @@ jobs:
|
|||||||
run: make -C ci conformance
|
run: make -C ci conformance
|
||||||
- name: Upload L2/L3 artifacts
|
- name: Upload L2/L3 artifacts
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||||
with:
|
with:
|
||||||
name: conformance-junit-coverage
|
name: conformance-junit-coverage
|
||||||
path: |
|
path: |
|
||||||
@@ -160,7 +160,7 @@ jobs:
|
|||||||
needs: [lint, l0, render-golden, l1, conformance]
|
needs: [lint, l0, render-golden, l1, conformance]
|
||||||
if: always()
|
if: always()
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
- name: Decide
|
- name: Decide
|
||||||
env:
|
env:
|
||||||
R_LINT: ${{ needs.lint.result }}
|
R_LINT: ${{ needs.lint.result }}
|
||||||
|
|||||||
@@ -60,9 +60,21 @@
|
|||||||
# entry's "What I did NOT verify" — that gap is stated, not papered over.
|
# entry's "What I did NOT verify" — that gap is stated, not papered over.
|
||||||
#
|
#
|
||||||
# OPERATOR PREREQUISITE — ONE SECRET, NOT COMMITTED HERE:
|
# OPERATOR PREREQUISITE — ONE SECRET, NOT COMMITTED HERE:
|
||||||
# `RENOVATE_TOKEN` — a Gitea access token for the bot account, scopes `write:repository`,
|
# `RENOVATE_TOKEN` — a Gitea access token for the bot account (`renovate-bot`) with FIVE scopes.
|
||||||
# `read:user` and (since 2026-10-29) `read:package`. Set it as an Actions secret on this
|
# Each is required by a call Renovate 44.x makes against Gitea 1.24.3, and Gitea refuses a
|
||||||
# repository. Nothing in this repo mints,
|
# token missing one with `403 token does not have at least one of required scope(s)`:
|
||||||
|
# read:user GET /user (platform init: initPlatform)
|
||||||
|
# read:organization GET /orgs/{owner}, /orgs/{o}/labels (initRepo -> isOrg, then org labels)
|
||||||
|
# write:repository clone/push branches, /pulls, /statuses
|
||||||
|
# write:issue /issues, /labels, /issues/{n}/labels (dependency dashboard, PR labels;
|
||||||
|
# Gitea files all of these under the ISSUE scope, not repository)
|
||||||
|
# read:package the Gitea container registry (RENOVATE_HOST_RULES below)
|
||||||
|
# CORRECTED 2026-10-30. This list used to name only write:repository + read:user (+ read:package),
|
||||||
|
# and the operator minted exactly that. Run 350 then failed at
|
||||||
|
# `GET /api/v1/orgs/Home-Network 403 ... required=[read:organization], token scope=read:package,
|
||||||
|
# write:repository,read:user`; write:issue would have been the next wall. The preflight below now
|
||||||
|
# PROBES these scopes, so a short token fails in seconds with the missing scope named.
|
||||||
|
# Set it as an Actions secret on this repository. Nothing in this repo mints,
|
||||||
# stores or references its value; it is consumed by name only.
|
# stores or references its value; it is consumed by name only.
|
||||||
# Until it exists, this workflow FAILS LOUDLY on every scheduled run. That is intended: a
|
# Until it exists, this workflow FAILS LOUDLY on every scheduled run. That is intended: a
|
||||||
# visible red is the correct state for "automation configured but not credentialed".
|
# visible red is the correct state for "automation configured but not credentialed".
|
||||||
@@ -161,10 +173,10 @@ jobs:
|
|||||||
That is the failure mode that let renovate.json sit inert in this repo for months while
|
That is the failure mode that let renovate.json sit inert in this repo for months while
|
||||||
seven files claimed "Renovate maintains this pin".
|
seven files claimed "Renovate maintains this pin".
|
||||||
|
|
||||||
TO FIX: create a Gitea access token for the bot account with scopes
|
TO FIX: create a Gitea access token for the bot account (renovate-bot) with scopes
|
||||||
`write:repository`, `read:user` and `read:package` (the last one lets Renovate read
|
read:user, read:organization, write:repository, write:issue and read:package (the
|
||||||
this repo's own images in the Gitea registry), then add it to this repository as the
|
header of .gitea/workflows/renovate.yaml says which call needs which), then add it to
|
||||||
Actions secret RENOVATE_TOKEN. Do not commit it anywhere.
|
this repository as the Actions secret RENOVATE_TOKEN. Do not commit it anywhere.
|
||||||
MSG
|
MSG
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -172,6 +184,75 @@ jobs:
|
|||||||
# Proves the step is running INSIDE the pinned Renovate image, not the default job image.
|
# Proves the step is running INSIDE the pinned Renovate image, not the default job image.
|
||||||
echo "renovate $(renovate --version)"
|
echo "renovate $(renovate --version)"
|
||||||
|
|
||||||
|
- name: Preflight - the token has every scope Renovate needs
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
|
||||||
|
# ADDED 2026-10-30. Renovate reports EVERY platform-init failure as the bare
|
||||||
|
# "Authentication failure" (lib/modules/platform/gitea/index.ts initPlatform swallows the
|
||||||
|
# HTTP error), and a later scope gap surfaces as "Repository has unknown error". Two
|
||||||
|
# dispatches were spent learning one missing scope at a time. These probes are GETs only,
|
||||||
|
# and they name the missing scope. The token never reaches argv: curl reads it from stdin
|
||||||
|
# (`--config -`), so it is not in `ps` output or in the log.
|
||||||
|
run: |
|
||||||
|
set -uo pipefail
|
||||||
|
GITEA=https://gitea.home.keithjackson.dev
|
||||||
|
REPO=Home-Network/home-network-k3s-impl
|
||||||
|
fail=0
|
||||||
|
body=$(mktemp)
|
||||||
|
probe() { # <scope it proves> <path> <auth: bearer|basic>
|
||||||
|
local code
|
||||||
|
if [ "$3" = basic ]; then
|
||||||
|
code=$(printf 'user = "renovate:%s"\n' "$RENOVATE_TOKEN" \
|
||||||
|
| curl -sS --config - -o "$body" -w '%{http_code}' "$GITEA$2") || code=000
|
||||||
|
else
|
||||||
|
code=$(printf 'header = "Authorization: Bearer %s"\n' "$RENOVATE_TOKEN" \
|
||||||
|
| curl -sS --config - -o "$body" -w '%{http_code}' "$GITEA$2") || code=000
|
||||||
|
fi
|
||||||
|
if [ "$code" = 200 ]; then
|
||||||
|
echo "ok $1 (GET ${2%%\?*} -> 200)"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
fail=1
|
||||||
|
echo "::error::scope $1: GET ${2%%\?*} -> $code"
|
||||||
|
case "$code" in
|
||||||
|
401) echo " 401 = Gitea rejected the token VALUE (wrong, expired or revoked)." ;;
|
||||||
|
403) if grep -q 'Access denied' "$body"; then
|
||||||
|
echo " 403 'Access denied' = a Cilium/Envoy policy denial, not Gitea."
|
||||||
|
else
|
||||||
|
echo " Gitea: $(jq -r '.message // empty' "$body" 2>/dev/null | head -c 300)"
|
||||||
|
fi ;;
|
||||||
|
*) echo " body: $(head -c 200 "$body")" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
probe read:user "/api/v1/user" bearer
|
||||||
|
probe read:organization "/api/v1/orgs/${REPO%%/*}" bearer
|
||||||
|
probe read:repository "/api/v1/repos/$REPO" bearer
|
||||||
|
probe read:issue "/api/v1/repos/$REPO/labels?limit=1" bearer
|
||||||
|
probe read:package "/v2/token?service=container_registry&scope=repository:home-network/toolbox:pull" basic
|
||||||
|
# WRITE scopes cannot be proven by a GET. Gitea's scope-refusal message, however, lists
|
||||||
|
# the token's scopes (names only, never the value): ask for an admin route this bot
|
||||||
|
# cannot have, and read `token scope=` from the 403. Advisory if the format changes.
|
||||||
|
printf 'header = "Authorization: Bearer %s"\n' "$RENOVATE_TOKEN" \
|
||||||
|
| curl -sS --config - -o "$body" "$GITEA/api/v1/admin/orgs?limit=1" || true
|
||||||
|
scopes=$(jq -r '.message // empty' "$body" 2>/dev/null | sed -n 's/.*token scope=//p')
|
||||||
|
if [ -n "$scopes" ]; then
|
||||||
|
echo "token scopes (from Gitea): $scopes"
|
||||||
|
for want in write:repository write:issue; do
|
||||||
|
case ",$scopes," in
|
||||||
|
*",$want,"*|*",all,"*) echo "ok $want" ;;
|
||||||
|
*) echo "::error::scope $want is missing (token scope=$scopes)"; fail=1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo "::warning::could not read the token's scope list; write:repository and write:issue are UNVERIFIED"
|
||||||
|
fi
|
||||||
|
rm -f "$body"
|
||||||
|
if [ "$fail" -ne 0 ]; then
|
||||||
|
echo "::error::RENOVATE_TOKEN failed the checks above (a 401 is the value, a 403 is a scope). Re-mint it for renovate-bot with read:user, read:organization, write:repository, write:issue, read:package and overwrite the secret."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Run Renovate
|
- name: Run Renovate
|
||||||
shell: bash
|
shell: bash
|
||||||
# The repo config is discovered by Renovate from the cloned repo (renovate.json5,
|
# The repo config is discovered by Renovate from the cloned repo (renovate.json5,
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: jackett
|
- name: jackett
|
||||||
image: lscr.io/linuxserver/jackett:latest
|
image: lscr.io/linuxserver/jackett:v0.24.2865-ls7
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 9117
|
- containerPort: 9117
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: pihole
|
- name: pihole
|
||||||
image: pihole/pihole
|
image: pihole/pihole:2026.10.0
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 80
|
- containerPort: 80
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: plex
|
- name: plex
|
||||||
image: plexinc/pms-docker:latest
|
image: plexinc/pms-docker:1.43.4.10903-e5521bd8c
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 32400
|
- containerPort: 32400
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: keycloak
|
- name: keycloak
|
||||||
image: quay.io/keycloak/keycloak
|
image: quay.io/keycloak/keycloak:26.0.6
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 8080
|
- containerPort: 8080
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
name: jackett
|
name: jackett
|
||||||
kind: container
|
kind: container
|
||||||
source:
|
source:
|
||||||
image: lscr.io/linuxserver/jackett:latest
|
image: lscr.io/linuxserver/jackett:v0.24.2865-ls7
|
||||||
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
||||||
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||||
storage:
|
storage:
|
||||||
@@ -70,7 +70,8 @@ auth:
|
|||||||
- ^/api/
|
- ^/api/
|
||||||
update:
|
update:
|
||||||
channel: stable # R-UPD-01
|
channel: stable # R-UPD-01
|
||||||
range: ">=0.22.0" # stay on the Jackett 0.22.x stable line
|
range: ">=0.22.0" # floor only, no ceiling (0.24.x runs as of 2026-10-30; Jackett has no
|
||||||
|
# major line to hold back from, so renovate.json5 mirrors no cap)
|
||||||
cadence: "0 4 * * *"
|
cadence: "0 4 * * *"
|
||||||
autoRollbackOnFailedHealth: true # R-UPD-03
|
autoRollbackOnFailedHealth: true # R-UPD-03
|
||||||
observability:
|
observability:
|
||||||
|
|||||||
@@ -5,8 +5,15 @@
|
|||||||
# Validates against schemas/servicedefinition.schema.json.
|
# Validates against schemas/servicedefinition.schema.json.
|
||||||
name: keycloak
|
name: keycloak
|
||||||
kind: container
|
kind: container
|
||||||
|
# DOCUMENTATION-ONLY IMAGE. The running server is NOT chosen here: the Keycloak operator
|
||||||
|
# (platform/keycloak/kustomization.yaml, three pinned release URLs) picks it via its
|
||||||
|
# RELATED_IMAGE_KEYCLOAK, and the Keycloak CR has no spec.image. This SD's render,
|
||||||
|
# apps/wave-1/keycloak.yaml, is not listed in apps/wave-1/kustomization.yaml and deploys nothing
|
||||||
|
# (ci/render-golden.sh RENDER_GOLDEN_NOT_ONBOARDED). The tag records the operator's version for
|
||||||
|
# readers; renovate.json5 disables Renovate for this image so it cannot raise PRs against a file
|
||||||
|
# nothing deploys. The Keycloak version moves with the operator — update this tag by hand then.
|
||||||
source:
|
source:
|
||||||
image: quay.io/keycloak/keycloak
|
image: quay.io/keycloak/keycloak:26.0.6
|
||||||
exposure: public # login/redirect flows for internet-exposed services; hardened (docs/15; R-DNS/R-NET)
|
exposure: public # login/redirect flows for internet-exposed services; hardened (docs/15; R-DNS/R-NET)
|
||||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||||
storage:
|
storage:
|
||||||
|
|||||||
+1
-1
@@ -5,7 +5,7 @@
|
|||||||
name: pihole
|
name: pihole
|
||||||
kind: container
|
kind: container
|
||||||
source:
|
source:
|
||||||
image: pihole/pihole
|
image: pihole/pihole:2026.10.0
|
||||||
exposure: internal # LAN-only admin UI (docs/13 row 6; R-DNS/R-NET)
|
exposure: internal # LAN-only admin UI (docs/13 row 6; R-DNS/R-NET)
|
||||||
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
egress: internet # normal WAN egress (docs/03; R-NET-05)
|
||||||
storage:
|
storage:
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@
|
|||||||
name: plex
|
name: plex
|
||||||
kind: container
|
kind: container
|
||||||
source:
|
source:
|
||||||
image: plexinc/pms-docker:latest
|
image: plexinc/pms-docker:1.43.4.10903-e5521bd8c
|
||||||
# plexinc/pms-docker env. Rendered VERBATIM onto the app container by render.py (schema env list).
|
# plexinc/pms-docker env. Rendered VERBATIM onto the app container by render.py (schema env list).
|
||||||
env:
|
env:
|
||||||
# PLEX_UID/PLEX_GID: the image boots as root (s6 PID1), chowns /config to this uid/gid, then drops
|
# PLEX_UID/PLEX_GID: the image boots as root (s6 PID1), chowns /config to this uid/gid, then drops
|
||||||
|
|||||||
+16
@@ -42,3 +42,19 @@ while IFS=: read -r wf _ value; do
|
|||||||
done < <(grep -nE '^[[:space:]]*configurationFile:' .gitea/workflows/*.y*ml 2>/dev/null || true)
|
done < <(grep -nE '^[[:space:]]*configurationFile:' .gitea/workflows/*.y*ml 2>/dev/null || true)
|
||||||
[ "$missing" -eq 0 ] || exit 1
|
[ "$missing" -eq 0 ] || exit 1
|
||||||
ok "workflow configurationFile references resolve"
|
ok "workflow configurationFile references resolve"
|
||||||
|
|
||||||
|
# A workflow must not use an action that needs a Docker daemon: jobs on this runner get NONE, by
|
||||||
|
# decision (decisions/2026-10-29-image-pipeline.md; act_runner maps docker_host to `-`). Such an
|
||||||
|
# action is valid YAML and passes every other stage, then dies at runtime with
|
||||||
|
# "failed to connect to the docker API at unix:///var/run/docker.sock". renovatebot/github-action
|
||||||
|
# did exactly that on every Renovate run until 2026-10-30 (run 343). Run the tool's image as the
|
||||||
|
# job `container:` instead (see .gitea/workflows/renovate.yaml). Container (`docker://`) actions
|
||||||
|
# and Docker's own daemon-bound actions are rejected for the same reason. Comments are ignored.
|
||||||
|
daemon_uses='uses:[[:space:]]*["'\'']?(docker://|renovatebot/github-action|docker/(build-push|setup-buildx|setup-qemu|login)-action)'
|
||||||
|
bad=0
|
||||||
|
while IFS= read -r hit; do
|
||||||
|
err "${hit%%:*}: needs a Docker daemon, which jobs here do not get: $(echo "${hit#*:*:}" | sed -E 's/^[[:space:]]+//')"
|
||||||
|
bad=1
|
||||||
|
done < <(grep -HnE "^[^#]*${daemon_uses}" .gitea/workflows/*.y*ml 2>/dev/null || true)
|
||||||
|
[ "$bad" -eq 0 ] || exit 1
|
||||||
|
ok "no workflow uses a Docker-daemon-bound action"
|
||||||
@@ -61,7 +61,7 @@ CONFORMANCE_RUNNER_IMAGE_TAG="chainsaw0.2.12-kubectl1.36.4-fc44"
|
|||||||
# chainsaw-job.yaml's @@CONFORMANCE_RUNNER_IMAGE_DIGEST@@ takes the 64 hex digits WITHOUT the
|
# chainsaw-job.yaml's @@CONFORMANCE_RUNNER_IMAGE_DIGEST@@ takes the 64 hex digits WITHOUT the
|
||||||
# `sha256:` prefix this value carries — the trigger must strip it.
|
# `sha256:` prefix this value carries — the trigger must strip it.
|
||||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/conformance-runner tag=main
|
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/conformance-runner tag=main
|
||||||
CONFORMANCE_RUNNER_IMAGE_DIGEST="sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
CONFORMANCE_RUNNER_IMAGE_DIGEST="sha256:65cc7c1ab576fe8cdf45e7f68c7cf4eb5ce7e0db46e62191f6190c51610b20d9"
|
||||||
CONFORMANCE_RUNNER_IMAGE="${CONFORMANCE_RUNNER_IMAGE_REPO}@${CONFORMANCE_RUNNER_IMAGE_DIGEST}"
|
CONFORMANCE_RUNNER_IMAGE="${CONFORMANCE_RUNNER_IMAGE_REPO}@${CONFORMANCE_RUNNER_IMAGE_DIGEST}"
|
||||||
#
|
#
|
||||||
# AFTERWARDS, TIGHTEN THE BOUNDARY: the header of conformance-trigger-job-shape.yaml asks for
|
# AFTERWARDS, TIGHTEN THE BOUNDARY: the header of conformance-trigger-job-shape.yaml asks for
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-06
|
||||||
|
branch: feat/pm-agent-registry-pull-secret
|
||||||
|
title: The kubelet's pull credential — the second half of making agent-runner pullable, and the seventh field the admission policy had not read
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
The registry had two blockers. The first — the nodes resolving the registry hostname to the
|
||||||
|
non-hairpinned WAN and failing TLS against the router's `CN=%D` certificate — was diagnosed
|
||||||
|
and fixed by an `/etc/hosts` pin, merged in #59 and applied by the operator. This is the
|
||||||
|
second: **the pull is unauthenticated, and this registry refuses unauthenticated clients
|
||||||
|
outright.**
|
||||||
|
|
||||||
|
`pm-agent-registry` is a new `kubernetes.io/dockerconfigjson` Secret, SOPS-encrypted with
|
||||||
|
placeholder values, referenced by `imagePullSecrets` on both the dispatcher Deployment and
|
||||||
|
the committed worker Job template. It is a **second Secret holding the same Gitea token** as
|
||||||
|
`pm-agent-credentials`, because the two consumers need two formats: the application reads a
|
||||||
|
raw token from env, the kubelet reads a dockerconfigjson before any container starts. That
|
||||||
|
duplication is the established pattern here (the Keycloak client secret is mirrored the same
|
||||||
|
way, for the same reason) and it is documented in both `.example` files.
|
||||||
|
|
||||||
|
The operator's part is the value, and one scope: the Gitea token must now carry
|
||||||
|
**`read:package`** alongside `write:repository` and `write:issue`. Steps are in
|
||||||
|
`platform/pm-agent/pm-agent-registry.sops.yaml.example`, including the trap that
|
||||||
|
`.dockerconfigjson`'s registry key must be the bare host as the image reference spells it —
|
||||||
|
with a scheme or a port it matches nothing, the kubelet pulls anonymously, and the failure is
|
||||||
|
indistinguishable from the Secret being absent.
|
||||||
|
|
||||||
|
**The admission policy gained a seventh rule.** `imagePullSecrets` is a pod-spec field that
|
||||||
|
no container references, which is exactly why it is easy to leave unread — the same shape of
|
||||||
|
omission as `initContainers`, `ephemeralContainers`, `env` and `envFrom` before it. The
|
||||||
|
kubelet reads the named Secret, in this namespace, before anything runs. Rule 1's pattern now
|
||||||
|
REQUIRES the committed entry and rule 7 forbids any other or any second one; both halves are
|
||||||
|
needed because a `foreach` over an absent list yields no result.
|
||||||
|
|
||||||
|
The leverage is small and worth saying so: the image is digest-pinned, so a different pull
|
||||||
|
credential cannot change *what* runs, and the only Secrets in this namespace are the agent's
|
||||||
|
own. It is pinned anyway, because "only the committed shape" is this policy's whole premise
|
||||||
|
and the rule costs four lines.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **`kyverno test platform/kyverno/tests/pm-agent` → 33 passed, 0 failed** under the pinned
|
||||||
|
CLI (was 28 before this change). Three new fixtures, each a real path rather than a style
|
||||||
|
violation: `extra-pull-secret` (the committed entry PLUS a second one — rule 1's array
|
||||||
|
pattern is satisfied by the first element, which is why the count is asserted separately),
|
||||||
|
`wrong-pull-secret` (a different Secret in this namespace), and `no-pull-secret` (absent
|
||||||
|
entirely, refused by rule 1's pattern because rule 7's foreach cannot see it).
|
||||||
|
- **`PM_SELFTEST=1 pm-dispatch` → exit 0**, now also asserting that the committed Job
|
||||||
|
template names exactly `pm-agent-registry` — so the template and the policy cannot drift
|
||||||
|
apart without the image build failing.
|
||||||
|
- **`./ci/gate.sh` in the pinned toolbox, from a §0 worktree → exit 3 (UNDETERMINED)**, with
|
||||||
|
`gitea-settings` the only stage that did not run (no `SETTINGS_API_TOKEN` on this
|
||||||
|
workstation). Every other stage green, `spec-immutability` included. rc 3 is not a pass.
|
||||||
|
- **The gate caught me shipping the Secret in plaintext.** I wrote the placeholder
|
||||||
|
`pm-agent-registry.sops.yaml` and never ran `sops -e -i` on it. `no-plaintext-secrets`
|
||||||
|
refused it with both halves named — "has no top-level `sops:` block with a mac" and
|
||||||
|
"Secret value '.dockerconfigjson' is not SOPS ciphertext" — which is exactly the check
|
||||||
|
earning its place: the file looked finished, and the value was a placeholder, so nothing
|
||||||
|
about reading the diff would have made it obvious. Encrypted, and the stage is green.
|
||||||
|
- **The registry really does refuse anonymous clients**, measured before building this
|
||||||
|
(2026-10-05): the anonymous registry token's `access` claim is `null`, every `/v2/…`
|
||||||
|
path answers 401 whether it exists or not, and `/api/v1/packages/Home-Network` replies
|
||||||
|
`{"message":"token is required"}`. That is why this is a credential and not a posture
|
||||||
|
tweak.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **That the pull now works.** Nothing has been pushed to that registry and the committed
|
||||||
|
digest is still the all-zero placeholder, so neither half of the registry fix has been
|
||||||
|
exercised end to end. The first real pull is what proves both at once — and if it fails, the
|
||||||
|
error names which half: `x509 … not valid for any names` is the resolution pin, `401` /
|
||||||
|
`unauthorized` is this credential.
|
||||||
|
- **That the `/etc/hosts` pin from #59 took effect.** The operator applied it; I could not
|
||||||
|
confirm the effect, because the dispatcher pod's retained events are all repeating
|
||||||
|
`BackOff` with no fresh `Pulling`, so the `x509` message still visible is the one from
|
||||||
|
before the play ran. Not evidence either way.
|
||||||
|
- **Whether `read:package` is sufficient** for a pull from a Gitea package of this type. It is
|
||||||
|
the documented read scope for packages; nothing here has exercised it, because no package
|
||||||
|
exists yet.
|
||||||
|
- **The dockerconfigjson placeholder is not a valid credential and is not meant to be.** Its
|
||||||
|
`auth` field is a `REPLACE-ME` string rather than base64, so a pull with it will fail at
|
||||||
|
authentication rather than be mistaken for working.
|
||||||
|
- **Anything about runtime.** Unchanged from the previous entries: the image has never been
|
||||||
|
built or pushed, no dispatcher has authenticated to Gitea, no issue has been triaged, no
|
||||||
|
worker Job has ever been created, and the admission policy — live, `Enforce`, now seven
|
||||||
|
rules — has still never admitted or refused a real Job.
|
||||||
|
|
||||||
|
**Addendum 2026-10-30 (rebased onto #81/#82/#83; see `decisions/2026-10-30-pm-agent-live.md`).**
|
||||||
|
Three statements above are now out of date, and one was wrong when written:
|
||||||
|
- *"Nothing has been pushed to that registry"* — no longer true. CI published agent-runner in
|
||||||
|
images run 340 (`sha-523cd9118f95` = `main`, digest `sha256:c4eba5b3…6eb42`), and this
|
||||||
|
branch now pins it.
|
||||||
|
- *"That the `/etc/hosts` pin from #59 took effect"* — now measured: all three prod nodes
|
||||||
|
carry the line, resolve the name to `192.168.2.20`, and get `401` with `ssl_verify_result=0`
|
||||||
|
from `https://gitea.home.keithjackson.dev/v2/` (see the new entry).
|
||||||
|
- *"Whether `read:package` is sufficient"* — the scope is necessary but not the whole answer:
|
||||||
|
org-package access is decided by account permission (team Packages unit, or org
|
||||||
|
visibility). Handled in the registry `.example`.
|
||||||
|
- **Wrong when written:** the scope list. Gitea 1.24.3 puts `GET /api/v1/user` — the
|
||||||
|
dispatcher's first call — behind the `user` scope category, so a token with only
|
||||||
|
`write:repository`, `write:issue` and `read:package` CrashLoops the dispatcher on startup.
|
||||||
|
The token needs `read:user` too.
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: feat/pin-pihole-jackett-plex
|
||||||
|
title: Pin pihole, jackett and plex to the digests already running, and make Renovate able to follow them
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- **`catalog/{pihole,jackett,plex}.yaml` `source.image`** pinned to the tag whose digest was
|
||||||
|
already running, so the *image* does not change:
|
||||||
|
- `pihole/pihole` (no tag) -> `pihole/pihole:2026.10.0`
|
||||||
|
- `lscr.io/linuxserver/jackett:latest` -> `lscr.io/linuxserver/jackett:v0.24.2865-ls7`
|
||||||
|
- `plexinc/pms-docker:latest` -> `plexinc/pms-docker:1.43.4.10903-e5521bd8c`
|
||||||
|
- **`apps/wave-0/{pihole,jackett,plex}.yaml`** re-rendered (`render.py --all catalog -o apps
|
||||||
|
--wave-layout --no-image-automation`, in the toolbox). The diff is exactly the three app-container
|
||||||
|
`image:` lines; nothing else moved.
|
||||||
|
- **`catalog/jackett.yaml`** `update.range` comment: "stay on the Jackett 0.22.x stable line" was
|
||||||
|
false (0.24 runs, and `>=0.22.0` has no ceiling). Range unchanged; comment corrected.
|
||||||
|
- **`renovate.json5`**, four packageRules plus a comment block:
|
||||||
|
- jackett `versioning: regex:^v(major).(minor).(patch)-ls(build)$` (Renovate's linuxserver example).
|
||||||
|
- jackett **`overridePackageName: ghcr.io/linuxserver/jackett`**: not in the brief, and needed.
|
||||||
|
Renovate's docker datasource reads at most `dockerMaxPages` (20, a global-only option) tag
|
||||||
|
pages from any host except a short list that includes ghcr.io (1000 pages). lscr.io is not on
|
||||||
|
that list. Jackett has **32,093** tags, oldest first, ~1000 per page; the running tag is #31,980.
|
||||||
|
Before this rule the dry run extracted jackett at `v0.24.2865-ls7` and **proposed nothing**,
|
||||||
|
with no warning. The override changes only the lookup; the manifests keep `lscr.io/...`.
|
||||||
|
Every jackett rule matches on `depName`, because the override rewrites `packageName`.
|
||||||
|
- jackett `schedule: after 3am and before 6am on sunday`, i.e. weekly. Measured: 13 ls builds in
|
||||||
|
12 days (ls7 built 2026-10-17, ls20 built 2026-10-29). Each is a `patch` and auto-merges, so on
|
||||||
|
the daily window it would mean a PR, a gate run and a jackett restart (Recreate) every day.
|
||||||
|
Weekly picks the newest tag, so it skips nothing. `catalog/jackett.yaml update.cadence` still
|
||||||
|
says daily. That field fed the retired Flux image-automation, and the rule's description
|
||||||
|
says the rule is now the cadence that takes effect.
|
||||||
|
- plex `versioning: regex:^(major).(minor).(patch).(build)-[0-9a-f]+$`.
|
||||||
|
- pihole: **no rule**, as the operator decided. CalVer parses under default docker versioning.
|
||||||
|
Patches and minors auto-merge, and a year rollover is a major (dashboard approval). This is
|
||||||
|
written in a comment.
|
||||||
|
- Corrected the "WHAT RENOVATE STILL CANNOT FIX" comment. It listed these three as floating;
|
||||||
|
they publish immutable version tags.
|
||||||
|
- No `allowedVersions` mirror was added. None of the three catalog ranges has a ceiling
|
||||||
|
(`>=2024.0.0`, `>=0.22.0`, `>=1.40.0`), so #86's mirror has nothing to encode for them.
|
||||||
|
- **Runbooks:** `image-policy-stalled.md` no longer says jackett and pms-docker are
|
||||||
|
`latest`-only. In `plex-quicksync.md`, "the floating-image caveat" became "the image-version
|
||||||
|
caveat", because the deployment no longer runs `:latest` with an implicit `Always`, and the symptom
|
||||||
|
is now "after a pms-docker bump". `plex-claim.md`'s two "running `:latest`" mentions are
|
||||||
|
now in the past tense.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Live digests** (read-only `kubectl --context k3s-prod`, pod `.status.containerStatuses[].imageID`):
|
||||||
|
pihole-0 and pihole-1 `sha256:d34134716fc3…`, jackett `sha256:4fcf3cc22dd3…`, plex
|
||||||
|
`sha256:e0ab27395614…`.
|
||||||
|
- **Each chosen tag resolves to the same digest** (anonymous registry HEAD, OCI index + manifest-list
|
||||||
|
Accept): `pihole/pihole:2026.10.0` = `sha256:d34134716fc3…`; `ghcr.io/linuxserver/jackett:v0.24.2865-ls7`
|
||||||
|
= `sha256:4fcf3cc22dd3…`; `plexinc/pms-docker:1.43.4.10903-e5521bd8c` = `sha256:e0ab27395614…`.
|
||||||
|
(For contrast, `pms-docker:latest` is now `sha256:6d778d41…` and `jackett:latest` `sha256:ee6ae915…`.
|
||||||
|
The pods would have drifted on their next restart.)
|
||||||
|
- `renovate-config-validator` (ghcr.io/renovatebot/renovate:44.167.0) reports "Config validated
|
||||||
|
successfully". The only warning is the existing `docker` block migration.
|
||||||
|
- **Renovate dry run** (`RENOVATE_PLATFORM=local`, `--dry-run=full`, 44.167.0, anonymous, includePaths
|
||||||
|
limited to the six files). It extracted all three from both `catalog/` (custom regex) and
|
||||||
|
`apps/wave-0/` (kubernetes) under the same depName, so each lands in one branch:
|
||||||
|
- plex `1.43.4.10903-e5521bd8c` -> `1.43.5.11079-b3b13e506` (patch, `renovate/plexinc-pms-docker-1.x`)
|
||||||
|
- jackett `v0.24.2865-ls7` -> `v0.24.2961-ls20` (patch, `renovate/lscr.io-linuxserver-jackett-0.x`).
|
||||||
|
This needs the ghcr override; without it there was no proposal.
|
||||||
|
- pihole `2026.10.0` -> none. That is correct, because 2026.10.0 is the newest numbered tag on Docker Hub.
|
||||||
|
- **Automerge, schedule and dashboard flags** come from Renovate's own `applyPackageRules`
|
||||||
|
(datasource-merge, then update-type-merge, the same order as `flatten.js`), run on the resolved
|
||||||
|
config. The local platform does not run the branch worker:
|
||||||
|
- plex patch: automerge **true**, daily window.
|
||||||
|
- jackett patch: automerge **true**, schedule **sunday** window.
|
||||||
|
- pihole (synthetic updates) `2026.10.1` patch and `2026.11.0` minor: automerge **true**. `2027.01.0`
|
||||||
|
major: automerge **false**, `dependencyDashboardApproval` **true**, +needs-review/breaking.
|
||||||
|
- **Versioning filters** (Renovate's versioning module): the jackett regex rejects `latest`, bare
|
||||||
|
`0.24.2961`, `amd64-…` and `development-…`. The plex regex rejects `latest`, `public`, `beta`,
|
||||||
|
`plexpass` and `…-amd64`/`…-arm64`. Docker versioning ranks `2026.09.0 < 2026.10.0`.
|
||||||
|
- Touched paths (`catalog/`, `apps/`, `renovate.json5`, `runbooks/`, `decisions/`) are not in
|
||||||
|
`infra/gitea/approval-policy.json` `sensitive_paths`.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **That the merge is restart-free. It is not.** The pod-template `image` string changes, so Flux
|
||||||
|
rolls all three: plex and jackett are `Recreate`, so each has a brief outage, and pihole is a
|
||||||
|
2-replica StatefulSet that rolls one pod at a time. The digest is identical, so the image's
|
||||||
|
behaviour should not change, but the kubelet resolves the new tag name and may pull it. The
|
||||||
|
rollout itself was not observed.
|
||||||
|
- The implicit `imagePullPolicy` moves from `Always` (`:latest` or no tag) to `IfNotPresent`. This is
|
||||||
|
intended, and was not observed live.
|
||||||
|
- A real (non-dry) Renovate run against Gitea, and whether the Sunday window actually opens a jackett
|
||||||
|
PR. The schedule was only evaluated, not exercised.
|
||||||
|
- Other `lscr.io/linuxserver/*` images (radarr, sonarr, qbittorrent) may hit the same 20-page
|
||||||
|
truncation once their tag counts grow. Not measured here, and not changed.
|
||||||
|
- Plex not self-updating on a version tag was checked only against the running pod (read-only
|
||||||
|
`kubectl exec cat`). `/etc/cont-init.d/50-plex-update` does `exit 0` when `version` from
|
||||||
|
`/version.txt` equals the installed `plexmediaserver` package. Both are
|
||||||
|
`1.43.4.10903-e5521bd8c`. I did not boot a pod on a *new* tag to watch that short-circuit.
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: feat/pm-agent-registry-pull-secret
|
||||||
|
title: PM agent — #61 rebased onto the CI image pipeline, first published digest pinned, and the remaining path to live audited
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
PR #61 (`feat/pm-agent-registry-pull-secret`, the kubelet pull credential and admission rule
|
||||||
|
7) was rebased onto `origin/main` after #81 (CI image pipeline), #82 and #83. Conflicts in
|
||||||
|
`docs/plans/pm-agent-issue-to-pr.md`, `platform/pm-agent/README.md` and
|
||||||
|
`runbooks/pm-agent.md` were resolved keeping both sides: #61's "pull credential now wired"
|
||||||
|
and #81's "CI publishes, Renovate bumps". No manual build-and-push text came back.
|
||||||
|
|
||||||
|
On top of the rebase, as a second commit:
|
||||||
|
|
||||||
|
- **The first published digest is pinned**: `deployment.yaml` and `PM_WORKER_IMAGE` in
|
||||||
|
`configmap.yaml` now name
|
||||||
|
`gitea.home.keithjackson.dev/home-network/agent-runner@sha256:c4eba5b3c508bf456a03391ac76f8275e7ea28a829da1450d99f6f6f25c6eb42`
|
||||||
|
(images run 340, tag `sha-523cd9118f95`, which is also `main`). The `# renovate-digest:`
|
||||||
|
markers are unchanged and still sit on the line directly above each value, so later bumps
|
||||||
|
stay Renovate's. Pinning by hand once removes the dependency on Renovate's first-ever run.
|
||||||
|
- **The first run is dry, and scoped to a label**: `PM_DRY_RUN: "true"`,
|
||||||
|
`PM_REQUIRE_LABEL: "agent"` (the plan's follow-up 3); `config-generation` bumped to `2`.
|
||||||
|
Dry run is also a *correctness* requirement, not only caution: the pinned digest was
|
||||||
|
built before this PR added `imagePullSecrets` to the worker Job template, which is baked
|
||||||
|
into the image. Admission now requires that entry, so a worker Job built from the run-340
|
||||||
|
image would be denied. Merging this PR touches `infra/agent/**`, so `images.yaml`
|
||||||
|
re-publishes agent-runner; Renovate's bump to that digest is what makes workers
|
||||||
|
admissible. Only then flip `PM_DRY_RUN` to `false`.
|
||||||
|
- **The token scope list was wrong; it is fixed.** Gitea 1.24.3's `routers/api/v1/api.go`
|
||||||
|
closes the `/user` route group with `tokenRequiresScopes(AccessTokenScopeCategoryUser)`,
|
||||||
|
and the dispatcher's first call is `GET /user`. The documented scopes are now
|
||||||
|
`read:user`, `write:repository`, `write:issue`, `read:package` everywhere (both
|
||||||
|
`.example`s, README, runbook row).
|
||||||
|
- **Org-package permission is documented separately from the scope.** Per
|
||||||
|
`services/context/package.go` `determineAccessMode` (v1.24.3), a non-admin's access to an
|
||||||
|
org-owned package is the highest Packages-unit mode across its org teams, else read if the
|
||||||
|
org is visible to it. Home-Network is `public`, so it works today by visibility alone; the
|
||||||
|
registry `.example` now has the operator create a `pm-agent-pull` team (no repositories,
|
||||||
|
Packages = Read) so it does not depend on that.
|
||||||
|
- **`sops` is toolbox-only**: both `.example`s now say so and give the in-place
|
||||||
|
`./infra/devcontainer/dev.sh run sops <file>` edit (the same procedure
|
||||||
|
`runbooks/vpn-remote-access.md` documents), which writes no plaintext file.
|
||||||
|
- The runbook's Secret-key check used `-o jsonpath='{.data}'`, which prints values before the
|
||||||
|
pipe trims them; it is now a `go-template` that emits key names only. Two new runbook
|
||||||
|
rows: the `read:user` 403, and the admission denial from a pre-#61 digest.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Registry digest**, read-only from the registry API (bearer token from `/v2/token`,
|
||||||
|
`HEAD /v2/home-network/agent-runner/manifests/<tag>` with OCI and Docker accept headers):
|
||||||
|
`main` and `sha-523cd9118f95` both answer `200`,
|
||||||
|
`Docker-Content-Digest: sha256:c4eba5b3…6eb42`, `application/vnd.oci.image.manifest.v1+json`.
|
||||||
|
Tags list: `[main, sha-523cd9118f95]`. The admission pattern
|
||||||
|
`gitea.home.keithjackson.dev/home-network/agent-runner@sha256:*` matches the pinned value.
|
||||||
|
- **#81's claim that the template's digest needs no tracking holds**: `pm-dispatch.build_job`
|
||||||
|
sets `containers[0].image = cfg.worker_image` (from `PM_WORKER_IMAGE`) on every Job.
|
||||||
|
Renovate's `renovate-digest` custom manager only matches `platform/pm-agent/*.yaml` and
|
||||||
|
`versions.env`, so the template is not touched by it.
|
||||||
|
- **Node path to the registry, read-only over SSH, all three prod nodes**: `/etc/hosts` line 9
|
||||||
|
is `192.168.2.20 keycloak.home.keithjackson.dev gitea.home.keithjackson.dev`, nsswitch is
|
||||||
|
`hosts: files dns`, and `curl https://gitea.home.keithjackson.dev/v2/` from the node answers
|
||||||
|
`401` with `ssl_verify_result=0` and `remote_ip=192.168.2.20`. That is the host resolver,
|
||||||
|
which is what containerd (a node process) uses; a pod's view would not have shown it.
|
||||||
|
- **Network path from the namespace**: dispatcher and worker both use
|
||||||
|
`PM_GITEA_URL=http://gitea.gitea.svc.cluster.local:3000` (ConfigMap, Job template, and
|
||||||
|
`pm-worker`'s hard allowlist, which refuses anything else); the git remote and the API base
|
||||||
|
are both derived from it. So nothing goes through the Gateway and #83's
|
||||||
|
`toEndpoints`-through-Gateway issue does not apply. Egress: `toEndpoints` gitea pods `:3000`
|
||||||
|
(both CNPs), `kube-apiserver` (dispatcher only), `world:443` (the model). Ingress on the
|
||||||
|
Gitea side: `gitea-allow-pm-agent-ingress` exists live and is `VALID True`.
|
||||||
|
- **Live state (read-only, `--context k3s-prod`)**: namespace `pm-agent` exists; Deployment
|
||||||
|
`0/1`, pod `ImagePullBackOff` for 24d on the all-zero digest; Secret `pm-agent-credentials`
|
||||||
|
(2 keys) exists, `pm-agent-registry` does not yet (it ships in this PR); Flux Kustomization
|
||||||
|
`pm-agent` Ready at `main@dfc432c`; ClusterPolicy `pm-agent-job-shape` Ready.
|
||||||
|
- **Gitea state (read-only API)**: 1.24.3; org Home-Network `public`; no user `pm-agent`
|
||||||
|
exists; teams are `Owners` and `registry-push`; collaborators are `jakman85` (owner) and
|
||||||
|
`renovate-bot` (write); the repository has **no labels** and **no open issues**.
|
||||||
|
- **Placeholders, without decrypting**: the registry ciphertext decodes to 155 bytes, exactly
|
||||||
|
the length of the `.example`'s placeholder JSON. The credentials values decode to 57 and 39
|
||||||
|
bytes — neither a 40-char Gitea token nor a ~100-char Claude OAuth token. Both files'
|
||||||
|
`lastmodified` is the commit that created them as placeholders.
|
||||||
|
- `kyverno test`, `PM_SELFTEST=1 pm-dispatch` and the gate: numbers in the PR description and
|
||||||
|
the final handoff report for this run.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **Any authenticated pull, poll or worker.** No `pm-agent` account or token exists, so the
|
||||||
|
scope set has not been exercised end to end. The `read:user` requirement is read from
|
||||||
|
Gitea's v1.24.3 source, not from a live 403.
|
||||||
|
- **That the credentials ciphertext is a placeholder.** Lengths are strong evidence, not
|
||||||
|
proof; I did not decrypt.
|
||||||
|
- **That Renovate will open the post-merge bump.** Its token was being re-minted at the time
|
||||||
|
of writing and it has never completed a run here. Renovate's `kubernetes` manager also
|
||||||
|
scans `infra/agent/*.yaml` and `platform/**`; what it does with a digest-only reference
|
||||||
|
with no tag (the template) is unmeasured.
|
||||||
|
- **The dispatcher's `claude --print` call to `api.anthropic.com`** through `world:443`, and
|
||||||
|
whether the run-340 image's dispatcher behaves as this tree's code does (it predates #61's
|
||||||
|
selftest line, nothing else in `pm-dispatch.py`).
|
||||||
|
- **The admission policy against a real Job.** Still never admitted or refused one live.
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: fix/renovate-auth
|
||||||
|
title: Renovate's token needs five scopes, not three; the workflow documented the wrong list, and now probes it
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
**The fix is operator-side.** `RENOVATE_TOKEN` lacks `read:organization`, and it will next lack
|
||||||
|
`write:issue`. The operator must re-mint it with five scopes:
|
||||||
|
|
||||||
|
- `read:user`
|
||||||
|
- `read:organization`
|
||||||
|
- `write:repository`
|
||||||
|
- `write:issue`
|
||||||
|
- `read:package`
|
||||||
|
|
||||||
|
The git half fixes the two things that sent the operator round this loop twice.
|
||||||
|
|
||||||
|
**`.gitea/workflows/renovate.yaml`**:
|
||||||
|
- **The header's scope list is corrected**, with the call each scope serves. The old list named
|
||||||
|
only `write:repository`, `read:user` and `read:package`, and the operator minted exactly that.
|
||||||
|
The preflight's "TO FIX" text is corrected the same way.
|
||||||
|
- **A new step, "Preflight - the token has every scope Renovate needs".**
|
||||||
|
- It probes read-only GETs: `/user` for read:user, `/orgs/Home-Network` for read:organization,
|
||||||
|
`/repos/{repo}` for read:repository, `/repos/{repo}/labels` for read:issue, and registry
|
||||||
|
`/v2/token` for read:package.
|
||||||
|
- It reads the token's own scope list out of Gitea's 403 from `/admin/orgs` (names only), and
|
||||||
|
requires `write:repository` and `write:issue` in it.
|
||||||
|
- It says what a failure means: 401 is the value; a Gitea 403 is a scope, and the probe prints
|
||||||
|
Gitea's message; an "Access denied" 403 is a Cilium/Envoy denial.
|
||||||
|
- The token is fed to curl on stdin (`--config -`), never in argv.
|
||||||
|
|
||||||
|
**`ci/lint.sh`**: a new guard rejects any non-comment `uses:` in `.gitea/workflows` that needs a
|
||||||
|
Docker daemon: `docker://`, `renovatebot/github-action` and Docker's daemon-bound
|
||||||
|
`docker/*-action`s. That is the class of bug fixed in #84.
|
||||||
|
|
||||||
|
### Root cause, with evidence
|
||||||
|
|
||||||
|
Run 350 (task 567, 06:49 UTC) **got further than 348 and 349.** Gitea's router log, filtered to
|
||||||
|
the act-runner pod `10.0.0.38`:
|
||||||
|
|
||||||
|
06:49:31 GET /api/v1/user 200 OK
|
||||||
|
06:49:31 GET /api/v1/version 200 OK
|
||||||
|
06:49:31 GET /api/v1/repos/Home-Network/home-network-k3s-impl 200 OK
|
||||||
|
06:49:31 GET /api/v1/orgs/Home-Network 403 Forbidden
|
||||||
|
|
||||||
|
There were no `/v2/` requests: Renovate stopped before extraction. From the job log, verbatim:
|
||||||
|
|
||||||
|
ERROR: Repository has unknown error (repository=Home-Network/home-network-k3s-impl)
|
||||||
|
"url": "https://gitea.home.keithjackson.dev/api/v1/orgs/Home-Network",
|
||||||
|
"message": "token does not have at least one of required scope(s), required=[read:organization], token scope=read:package,write:repository,read:user"
|
||||||
|
"x-envoy-upstream-service-time": "27", "server": "envoy"
|
||||||
|
INFO: Repository finished ... "result": "unknown-error", "exitCode": 8
|
||||||
|
|
||||||
|
**This is not Cilium.** The body is Gitea's JSON scope message, Gitea logged the request, and
|
||||||
|
`x-envoy-upstream-service-time` shows that Envoy forwarded it upstream.
|
||||||
|
|
||||||
|
**The secret WAS updated.** Runs 348 and 349 got `GET /user 403` from `tokenRequiresScopes`. Run 350
|
||||||
|
got `/user 200`, and its token scope list now includes `read:user`.
|
||||||
|
|
||||||
|
**Why `read:organization` is needed.** Renovate 44.167.0, `lib/modules/platform/gitea/index.ts`
|
||||||
|
`initRepo`, calls `helper.isOrg` → `GET /orgs/{owner}` and rethrows on error. Gitea 1.24.3,
|
||||||
|
`routers/api/v1/api.go`, puts `/orgs/{org}` behind
|
||||||
|
`tokenRequiresScopes(AccessTokenScopeCategoryOrganization)`.
|
||||||
|
|
||||||
|
**Why `write:issue` is needed next.** In the same `api.go`, `/repos/{o}/{r}/issues…`, `/labels`
|
||||||
|
and `/milestones` sit in a group with `tokenRequiresScopes(AccessTokenScopeCategoryIssue)`
|
||||||
|
(lines 1466–1582). Renovate's gitea-helper uses `/issues`, `/issues/{n}/labels`, `/labels` and
|
||||||
|
`/issues/{n}/comments`, for the dependency dashboard (`:dependencyDashboard` is extended) and for
|
||||||
|
PR labels. A dry run only reads them; a scheduled run writes them.
|
||||||
|
|
||||||
|
The other causes the coordinator listed are ruled out:
|
||||||
|
- **Schedule window and dry-run.** The run never reached branch processing.
|
||||||
|
- **Repository config.** The run never reached the clone (`"cloned": undefined`).
|
||||||
|
- **The Summary step.** It is not involved; the failure is Renovate's own exit 8.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- I read the whole run 350 job log (204 lines), and the router lines above. Runs 348 and 349 are
|
||||||
|
as recorded in `decisions/2026-10-30-renovate-first-run.md`'s follow-up: `/user` got 403 from
|
||||||
|
`tokenRequiresScopes`.
|
||||||
|
- **The probe step, in the pinned image.** I extracted the step body from the YAML and ran it in
|
||||||
|
`ghcr.io/renovatebot/renovate:44.167.0` (`curl` 8.5.0 and `jq` are present) with a deliberately
|
||||||
|
**bogus** token. Every probe reported 401 with the "token VALUE" diagnosis, the scope-list read
|
||||||
|
fell back to a warning, and the step exited 1.
|
||||||
|
- **The scope-list parser.** Against run 350's exact message format it reports `write:issue`
|
||||||
|
MISSING. Against a five-scope list it reports both OK. Against `all` it reports both OK.
|
||||||
|
- **The lint guard.** It flags `renovatebot/github-action` in the pre-#84 `renovate.yaml`
|
||||||
|
(dfc432c, line 152). It reports nothing on the current workflows, and it ignores commented
|
||||||
|
lines. `bash -n ci/lint.sh` passes.
|
||||||
|
- **Account facts, read-only.** `renovate-bot` is a collaborator with `write`, and the org is
|
||||||
|
public.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **The probe with a real token. UNVERIFIED.** I never had the token. In particular, I have not
|
||||||
|
observed that `GET /api/v1/admin/orgs` gives renovate-bot's token a 403 with a `token scope=`
|
||||||
|
list. The source says the scope check runs before `reqSiteAdmin`. If the message format
|
||||||
|
differs, the step warns and does not fail.
|
||||||
|
- **That five scopes are enough for a full non-dry run.** They cover every helper endpoint Renovate
|
||||||
|
44.167.0 calls, mapped to Gitea 1.24.3's route categories by reading source. Branch pushes go
|
||||||
|
over git-over-HTTPS with the token; I believe that needs write:repository, but I have not
|
||||||
|
observed it.
|
||||||
|
- **Registry reads as renovate-bot.** `read:package` on a public org is expected to work. The new
|
||||||
|
probe is the first real test of it.
|
||||||
|
- **Sensitive paths.** `.gitea/**` and `ci/**` are both sensitive. CI's `approval-policy` will block
|
||||||
|
until the operator posts `/approve-override` as a PR review.
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: fix/renovate-hold-auth-and-self
|
||||||
|
title: Operator decision — the oauth2-proxy sidecar and Renovate's own version never auto-merge
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
`renovate.json5` gains two packageRules. Both set `automerge: false` and `addLabels: ["needs-review"]`.
|
||||||
|
They sit after the auto-merge rule, so they override it at every update type. Each rule's
|
||||||
|
`description` carries the operator's reason.
|
||||||
|
|
||||||
|
1. **`quay.io/oauth2-proxy/oauth2-proxy`.** This is the single depName emitted for:
|
||||||
|
- `catalog/render.py` `OAUTH2_PROXY_IMAGE` (custom regex manager);
|
||||||
|
- the five rendered `apps/wave-*` sidecars;
|
||||||
|
- `platform/crafty/deployment.yaml` (kubernetes manager).
|
||||||
|
|
||||||
|
**Reason:** since v7.13.0, oauth2-proxy validates refreshed OIDC sessions against the access
|
||||||
|
token, not the id_token. Every sidecar sets `--cookie-refresh`, and that path is untested against
|
||||||
|
this Keycloak: the platform oauth2-proxy runs v7.15.5 without `--cookie-refresh`.
|
||||||
|
`render-golden` and `--selftest` pass on the bump, but both check rendered text, not the binary.
|
||||||
|
2. **`ghcr.io/renovatebot/renovate`.** This is the job `container.image` in
|
||||||
|
`.gitea/workflows/renovate.yaml`. The rule also lists `renovatebot/github-action`, so a
|
||||||
|
reintroduction would be held too. It is not used anywhere today; `ci/lint.sh` rejects it.
|
||||||
|
|
||||||
|
**Reason:** nothing re-validates `renovate.json5` against the new version before it takes
|
||||||
|
effect. The 39.x `CONFIG_VALIDATION` failure is the precedent.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **The config validates on the pinned 44.167.0.** `renovate-config-validator` printed
|
||||||
|
`Config validated successfully`. The migration diff is only the pre-existing top-level `docker`
|
||||||
|
block.
|
||||||
|
- **The flag change is exactly the intended one.** I used Renovate 44.167.0's own
|
||||||
|
`resolveConfigPresets` and `applyPackageRules` to evaluate b4ff287's config and this branch's
|
||||||
|
config against the same updates (the local dry run of the #86 config). Exactly **2 of 29**
|
||||||
|
branch rows differ:
|
||||||
|
- `ghcr.io-renovatebot-renovate-44.x`: `automerge=true labels=` → `automerge=false labels=needs-review`;
|
||||||
|
- `quay.io-oauth2-proxy-oauth2-proxy-7.x`: the same change, and every upgrade in the branch
|
||||||
|
resolves `false`, the `catalog/render.py` one included.
|
||||||
|
|
||||||
|
The other 27 rows are byte-identical.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **The three GitHub-hosted branches** in run 357 (`pin-dependencies`, `kubevirt-cdi`,
|
||||||
|
`intel-intel-device-plugins-for-kubernetes-0.x`) are not in the harness table: my local run has
|
||||||
|
no github.com token. Neither rule's package names can match them.
|
||||||
|
- **A live Renovate run with this config.** The next debug-level dispatch should show
|
||||||
|
`needs-review` on both branches.
|
||||||
|
- **The oauth2-proxy refresh path itself** remains untested. Exercising it on one app is the
|
||||||
|
review the label asks for.
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: fix/renovate-run-hygiene
|
||||||
|
title: Renovate's first completed dry runs, audited; it would have opened red PRs for every app and missed the databases. Config fixed
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
**`renovate.json5` only.** Each change carries a comment quoting the measurement behind it.
|
||||||
|
|
||||||
|
1. **Three new regex managers, so updates edit the SOURCE.**
|
||||||
|
- `catalog/*.yaml` `source.image`.
|
||||||
|
- `catalog/render.py` `OAUTH2_PROXY_IMAGE`.
|
||||||
|
- CNPG `imageName:` under `platform/` and `apps/`.
|
||||||
|
|
||||||
|
The first two match the rendered line's depName, so the catalog edit and the generated
|
||||||
|
`apps/wave-*` edit land in **one** branch.
|
||||||
|
2. **`allowedVersions` mirroring each ServiceDefinition's `update.range`:**
|
||||||
|
- gitea/gitea `<2.0.0`
|
||||||
|
- nextcloud `<32.0.0`
|
||||||
|
- qbittorrent `<6.0.0`
|
||||||
|
- radarr `<6.0.0`
|
||||||
|
- sonarr `<5.0.0`
|
||||||
|
- semaphore `<3.0.0`
|
||||||
|
|
||||||
|
Plus CNPG postgresql `<17.0.0`, because a Postgres major is a migration, not a bump.
|
||||||
|
3. **The blast-radius never-automerge list gains** `gitea/gitea`, `gitea/act_runner` and
|
||||||
|
`ghcr.io/cloudnative-pg/postgresql`.
|
||||||
|
4. **HelmRelease range widening** (`matchManagers: flux`, ranged currentValue) is never
|
||||||
|
auto-merged and is labelled `range-widen`.
|
||||||
|
5. **Majors need a Dependency Dashboard tick** (`dependencyDashboardApproval`) before a PR opens.
|
||||||
|
6. **`flux` manager docker deps are disabled.** These are duplicates of `kubernetes` manager
|
||||||
|
extractions.
|
||||||
|
7. **`velero/velero-plugin-for-aws` uses `semver` versioning**, so its RC pin can reach a stable
|
||||||
|
release.
|
||||||
|
8. **`python` minors are not auto-merged.**
|
||||||
|
9. **Two false comments are corrected.**
|
||||||
|
- Ranges "cap Renovate at patches": false, Renovate widens them.
|
||||||
|
- `nextcloud:31-apache` and `docker:27-dind` "cannot be maintained": false, they were.
|
||||||
|
|
||||||
|
PR #85 is merged, so nothing here overlaps an open branch.
|
||||||
|
|
||||||
|
### The audit: dispatch runs 353 (07:12 UTC) and 354 (07:23 UTC), on 593d0fa
|
||||||
|
|
||||||
|
Both runs ended `success`, Renovate `"result": "done"`, `exitCode 0`. Both were at `info` level, so
|
||||||
|
I reproduced the per-dependency detail with a debug-level local dry run of the same commit (see
|
||||||
|
below).
|
||||||
|
|
||||||
|
The runs were inside the 03:00–06:00 America/New_York window, so branches were processed:
|
||||||
|
- run 353: 37 "Would commit" branches;
|
||||||
|
- run 354: 40, because the operator had added `RENOVATE_GITHUB_COM_TOKEN` 9 s before it.
|
||||||
|
|
||||||
|
Gitea's router log for the act-runner pod during both runs shows 317 2xx responses. The only
|
||||||
|
non-2xx were all expected:
|
||||||
|
- `/v2/` 401: the registry challenge;
|
||||||
|
- `/admin/orgs` 403: the #85 scope probe, by design;
|
||||||
|
- 6× `HEAD …/manifests/sha256:000…` 404: the placeholder digests.
|
||||||
|
|
||||||
|
| # | Class | Count | Cause | Severity | Fix |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 1 | App-image PRs edit only generated `apps/wave-*` | 9 branches | A ServiceDefinition has no `apiVersion`, so the `kubernetes` manager extracts 0 deps from `catalog/`. The oauth2-proxy sidecar is a Python constant (`catalog/render.py:132`) | **Real problem.** Every such PR fails `ci/render-golden.sh` (reproduced: rc 1 "GOLDEN DRIFT"), stays red and holds a PR slot | git (1) |
|
||||||
|
| 2 | CNPG `imageName` untracked | 3 Clusters | The `kubernetes` manager reads `image:` only. The one tracked postgres line was the grant Job (`nextcloud-db.yaml:303`, "keep in step with spec.imageName") | **Real problem.** A real run bumps the Job and leaves the databases behind | git (1) |
|
||||||
|
| 3 | Bogus or out-of-policy majors | 5 branches: gitea 28.2.0, qbittorrent 20.04.1, radarr 6.x, nextcloud 35, postgresql 18.6 | Odd legacy tags, plus `update.range` in `catalog/` not being read by Renovate | Risk if run for real: unmergeable PRs, which hold slots | git (2) |
|
||||||
|
| 4 | Auto-merge of the merge gate's own host | gitea 1.24.3→1.27.3, act_runner 0.2.11→0.6.1, both "minor" | Not in the blast-radius list. A 0.x minor is breaking by semver | **Risk if run for real** | git (3) |
|
||||||
|
| 5 | Range widening auto-merged | external-dns, headlamp (minor); longhorn (blast radius); kube-prometheus-stack (major) | Renovate widens `>=a <b`. The config's comment claimed the opposite | **Risk if run for real:** silently lets Flux adopt new minors | git (4, 9) |
|
||||||
|
| 6 | Majors hold PR slots | 10 majors in run 354 against `prConcurrentLimit: 5` | Majors never auto-merge, so they stay open | Risk: starves auto-mergeable PRs | git (5) |
|
||||||
|
| 7 | `Cannot find replaceString … alpine:3.22` (INFO) | 1 per run | The same HelmRelease `image:` line is extracted by `flux` and `kubernetes` (also the velero plugin) | Cosmetic. The branch is still right | git (6) |
|
||||||
|
| 8 | velero-plugin-for-aws stuck on `v1.14.3-rc.2` | 1 dep, 0 updates | Docker versioning treats `-rc.2` as a suffix, so stable `v1.14.3` and `v1.14.4` never qualify | **Real problem** (silent) | git (7) |
|
||||||
|
| 9 | python 3.13→3.15 as auto-mergeable "minor" | 2 files | Language runtime minor | Risk if run for real | git (8) |
|
||||||
|
| 10 | `WARN GitHub token is required` | run 353: 5 packages, 17 deps | No github.com token | Real (they were unmaintained) | **operator, done:** run 354 has no WARN and adds `pin-dependencies`, `kubevirt-cdi`, `actions-checkout-7.x`, `major-github-artifact-actions` |
|
||||||
|
| 11 | Floating or untagged images (`invalid-value`) | 4: jackett, plex `latest`; pihole, keycloak no tag | No version to compare | Real, already documented | none in git. Pinning by hand is an operator/PM decision |
|
||||||
|
| 12 | `runs-on: ubuntu-latest` `invalid-version` | 11 | The github-actions manager extracts runner labels | Cosmetic, debug-only | none |
|
||||||
|
| 13 | `Docker Manifest is unknown` (debug) | 3 | HEAD on the all-zero placeholder digests | Expected. Clears when the own-image digest PRs merge | none |
|
||||||
|
| 14 | `Docker: error fetching data from DockerHub` 403 (debug) | 5 | Hub refuses `page=11`. Renovate already has the tags | Cosmetic. Updates were still found | none |
|
||||||
|
| 15 | `Config migration necessary` | 1 | Top-level `"docker": {"enabled": true}` | Cosmetic, pre-existing | none (as before) |
|
||||||
|
|
||||||
|
**Checked and fine:**
|
||||||
|
- The 3 own-image digest updates were found in both runs. All 4 `# renovate-digest:` markers parse.
|
||||||
|
- No marker failed to parse.
|
||||||
|
- No package file is under `conformance/runs` or `tests`, nor is `DECISIONS-LOG.md`, `decisions/` or
|
||||||
|
any `*.sops.yaml` (77 package files checked).
|
||||||
|
- `kubevirt-cdi` grouping works.
|
||||||
|
- The schedule window behaved as designed.
|
||||||
|
- Vulnerability alerts: none in either log.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- I read both job logs in full (132 and 129 lines), and the router log for 10.0.0.38 during both
|
||||||
|
runs.
|
||||||
|
- **`renovate-config-validator` on 44.167.0**: `Config validated successfully`. The migration diff
|
||||||
|
is only the pre-existing `docker` block.
|
||||||
|
- **Local dry runs.** Pinned image, `--platform=local`, debug, origin/main snapshot. "Before" is
|
||||||
|
593d0fa and "after" is this config; the GitHub-hosted deps were skipped in both.
|
||||||
|
- **Extraction:** `regex` went from 6 files / 7 deps to **19 files / 20 deps**. Every other
|
||||||
|
manager is unchanged: flux 15, github-actions 24, kubernetes 131, kustomize 2.
|
||||||
|
- **Branches:** 33 before, **29 after**. Removed: gitea-28.x, qbittorrent-20.x, radarr-6.x,
|
||||||
|
nextcloud-35.x and postgresql-18.x. Added: velero `v1.14.3-rc.2 → v1.14.4` (patch).
|
||||||
|
- **Gitea, qbittorrent, radarr, sonarr, semaphore and seerr branches** now each carry
|
||||||
|
`catalog/<svc>.yaml` + `apps/wave-*/<svc>.yaml`.
|
||||||
|
- **The oauth2-proxy branch** carries `catalog/render.py` plus the 5 rendered apps and
|
||||||
|
`platform/crafty`.
|
||||||
|
- **The postgresql-16.x branch** carries all three Clusters.
|
||||||
|
- The 2 flux docker duplicates report `disabled`.
|
||||||
|
- **render==golden holds for the new branch shape.**
|
||||||
|
- In the toolbox, I applied the gitea branch edit (catalog + apps) and the oauth2-proxy branch
|
||||||
|
edit (`render.py` constant + 5 apps) textually. `./ci/render-golden.sh` reported
|
||||||
|
`all 10 onboarded service golden(s) reproduce`, rc 0.
|
||||||
|
- Negative control, the old shape with only `apps/wave-0/gitea.yaml` bumped: rc 1,
|
||||||
|
`GOLDEN DRIFT`.
|
||||||
|
- Both edits were reverted afterwards.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **The effect of the automerge, label and dashboard-approval rules (3, 4, 5, 8). UNVERIFIED in a
|
||||||
|
run.** `--platform=local` stops before branch processing, so I saw only which updates exist, not
|
||||||
|
their merged flags. The matchers are standard (`matchPackageNames`, `matchManagers`,
|
||||||
|
`matchCurrentValue`, `matchUpdateTypes`). The next dispatch's debug log shows them per branch.
|
||||||
|
- **`allowedVersions` under docker versioning** used Renovate's "Falling back to npm semver
|
||||||
|
syntax" path (16 debug lines). That it filters is shown by the five removed branches.
|
||||||
|
- **render==golden for the other app branches** (qbittorrent, radarr, sonarr, semaphore, seerr). I
|
||||||
|
proved it only for gitea and oauth2-proxy. They have the same shape: one `source.image` line ↔
|
||||||
|
one rendered `image:` line.
|
||||||
|
- **The `allowedVersions` rules duplicate `catalog/*/update.range`** and can drift from it. Nothing
|
||||||
|
checks them against each other. A lint comparing the two is the obvious follow-up.
|
||||||
|
- **Any real (non-dry) run.** None has happened. PR creation, labels, auto-merge through Gitea's
|
||||||
|
API and the Dependency Dashboard issue are all unobserved.
|
||||||
|
- **Sensitive path?** `renovate.json5` is not in `infra/gitea/approval-policy.json`, as far as I
|
||||||
|
know. CI's `approval-policy` result on the PR is the measurement.
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-30
|
||||||
|
branch: feat/renovate-track-keycloak-operator
|
||||||
|
title: Renovate tracks the Keycloak operator (the real Keycloak version), never auto-merged; the catalog's phantom image is pinned as documentation and disabled
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- **`platform/keycloak/kustomization.yaml`**: each of the three `keycloak-k8s-resources/26.0.6`
|
||||||
|
release URLs (two CRDs and `kubernetes.yml`) now has a
|
||||||
|
`# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker on the line
|
||||||
|
above it. That is the format the existing `custom.regex` marker manager parses (the kubevirt
|
||||||
|
pattern). A comment explains that these URLs are the Keycloak version. The `Keycloak` CR has no
|
||||||
|
`spec.image`, so the operator's `RELATED_IMAGE_KEYCLOAK` picks the server image.
|
||||||
|
- **`renovate.json5`** has two new packageRules:
|
||||||
|
- **`keycloak/keycloak-k8s-resources`**: `groupName: keycloak-operator`, `automerge: false`,
|
||||||
|
`addLabels: [needs-review, blast-radius]`. A comment explains why. The operator, the CRDs and
|
||||||
|
the server are released in lockstep. The IdP gates SSO for every app and for kubectl. And
|
||||||
|
`passkey-reconciler.yaml` encodes Keycloak 26.0.6 Admin API shapes that the static gate never
|
||||||
|
exercises. Majors still go through the existing dashboard-approval rule.
|
||||||
|
- **`quay.io/keycloak/keycloak`**: `enabled: false`. Its only occurrences are in files that
|
||||||
|
deploy nothing.
|
||||||
|
|
||||||
|
The "STILL CANNOT FIX" comment also gains a one-line correction for keycloak.
|
||||||
|
- **`catalog/keycloak.yaml`**: `source.image` changed from `quay.io/keycloak/keycloak` (no tag) to
|
||||||
|
`quay.io/keycloak/keycloak:26.0.6`. A comment says the image is DOCUMENTATION-ONLY and that the
|
||||||
|
operator chooses the real one. `apps/wave-1/keycloak.yaml` was re-rendered, which changes one
|
||||||
|
line.
|
||||||
|
- **`platform/keycloak/README.md`**:
|
||||||
|
- Corrects the false claim "Renovate maintains the version". Until now Renovate could not see
|
||||||
|
the version.
|
||||||
|
- Adds a new section, **"Upgrading Keycloak: the operator IS the version"**. It lists the 26.0.6
|
||||||
|
assumptions in the passkey reconciler to diff against a new tag, and the post-merge checks.
|
||||||
|
|
||||||
|
**The phantom: pin and disable, not delete.** `apps/wave-1/keycloak.yaml` is not listed in
|
||||||
|
`apps/wave-1/kustomization.yaml`, and render-golden excludes it (`RENDER_GOLDEN_NOT_ONBOARDED`).
|
||||||
|
Deleting it would still not be a no-op:
|
||||||
|
|
||||||
|
- It is the fixture for the keycloak cases in `apps/wave-1/tests/app-slis.test.yaml` (the
|
||||||
|
StatefulSet-based `keycloak-down` rule). Those cases are discovered by the
|
||||||
|
every-PrometheusRule-in-apps/wave-1 convention.
|
||||||
|
- Removing it means moving that rule and its tests to `platform/keycloak`. That is a separate
|
||||||
|
observability change, not a Renovate one.
|
||||||
|
|
||||||
|
Pinning plus `enabled: false` changes one line in a file nothing applies. It cannot affect the
|
||||||
|
cluster, and render-golden stays green.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Live state, prod, read-only kubectl, 2026-10-30.**
|
||||||
|
- Keycloak CR `keycloak` has an empty `spec.image`.
|
||||||
|
- The operator Deployment runs `quay.io/keycloak/keycloak-operator:26.0.6`, imageID
|
||||||
|
`sha256:8e3a1a56346f…`, with `RELATED_IMAGE_KEYCLOAK=quay.io/keycloak/keycloak:26.0.6`.
|
||||||
|
- `keycloak-0` and `keycloak-1` both run `quay.io/keycloak/keycloak:26.0.6`, imageID
|
||||||
|
`sha256:a93d22e13b86…`.
|
||||||
|
- There is no PrometheusRule containing "keycloak" in any namespace. This confirms the phantom
|
||||||
|
deploys nothing, including its `keycloak-down` rule (see the next section).
|
||||||
|
- **`renovate-config-validator`** on the pinned `ghcr.io/renovatebot/renovate:44.167.0`:
|
||||||
|
`Config validated successfully`. The only migration note is the pre-existing `docker` block.
|
||||||
|
- **Local dry run** (`RENOVATE_PLATFORM=local`, `RENOVATE_DRY_RUN=full`, debug, the same pinned
|
||||||
|
image, this worktree, with a GitHub token passed by environment variable; the token string
|
||||||
|
occurs 0 times in the log):
|
||||||
|
- The regex manager went to 21 files / 24 deps.
|
||||||
|
- `keycloak/keycloak-k8s-resources` was extracted three times from
|
||||||
|
`platform/keycloak/kustomization.yaml`, each at `currentValue 26.0.6`, datasource
|
||||||
|
`github-tags`, versioning `semver`.
|
||||||
|
- Each extraction has one update, `26.8.1`, updateType `minor`, and all three share **one**
|
||||||
|
`branchName: renovate/keycloak-operator`.
|
||||||
|
- `quay.io/keycloak/keycloak` was extracted from `catalog/keycloak.yaml` (regex) and from
|
||||||
|
`apps/wave-1/keycloak.yaml` (kubernetes) at 26.0.6, with `skipReason: disabled` and no
|
||||||
|
updates.
|
||||||
|
- **Rule effects.** `--platform=local` does not print each branch's merged flags. So I ran
|
||||||
|
Renovate's own `applyPackageRules` (from `dist/util/package-rules` in the same 44.167.0 image)
|
||||||
|
over this `renovate.json5` for each case:
|
||||||
|
|
||||||
|
| Update | Result |
|
||||||
|
|---|---|
|
||||||
|
| operator minor 26.0.6 → 26.8.1 | `automerge:false`, `groupName:keycloak-operator`, labels `dependencies, automated-update, needs-review, blast-radius` |
|
||||||
|
| operator patch | same as the minor |
|
||||||
|
| operator major | `automerge:false`, `dependencyDashboardApproval:true`, labels add `breaking` |
|
||||||
|
| `quay.io/keycloak/keycloak`, catalog and rendered | `enabled:false` |
|
||||||
|
| control: sonarr patch | `automerge:true`, so the rule did not disable auto-merge globally |
|
||||||
|
- **render-golden, host:** `render.py --selftest` passed, and `all 10 onboarded service golden(s)
|
||||||
|
reproduce`. Before the pin, the committed phantom was byte-identical to the canonical render. I
|
||||||
|
re-rendered it after the pin.
|
||||||
|
- **Toolbox gate** (commit 24db269, worktree with the ENGINE shim mounting `.git` and the spec
|
||||||
|
repo, `DEV_KUBECONFIG=/dev/null`): **rc 3**.
|
||||||
|
- The only stage that did not run is **`gitea-settings`**.
|
||||||
|
- spec-immutability, decisions-log, lint, l0, render-golden, l1 and approval-policy all passed.
|
||||||
|
- L2/L3 `conformance` scaffold-skipped, as expected.
|
||||||
|
- **Not green.**
|
||||||
|
- **Sensitive paths:** none touched (checked against `infra/gitea/approval-policy.json`), so no
|
||||||
|
`/approve-override` is needed.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **A real Renovate run.** No PR was opened by Renovate. The labels and `automerge:false` are
|
||||||
|
proven by Renovate's own matcher, not by an opened PR.
|
||||||
|
- **That 26.8.1 is safe to adopt.** I did not diff 26.0.6 against 26.8.1. That includes the
|
||||||
|
reconciler's assumptions: `recurseExecutions` displayName, `updateExecutions`, the
|
||||||
|
`RpEntityName` guard, `requiredActions` REPLACE, and config aliases. The README section lists
|
||||||
|
them for whoever reviews the first bump PR. It also says to take or confirm a fresh CNPG backup
|
||||||
|
first, because a server minor may migrate the schema.
|
||||||
|
- **FINDING, not fixed: Keycloak has no deployed "IdP down" alert.** The `keycloak-down` SLI rule
|
||||||
|
exists only inside the undeployed `apps/wave-1/keycloak.yaml`. promtool tests it, but it is not
|
||||||
|
on the cluster: there is no keycloak PrometheusRule live. Separately, because that file exists,
|
||||||
|
`render.py --emit-waves` would treat keycloak as onboarded. It would then add a Deployment named
|
||||||
|
`keycloak` beside the operator's StatefulSet (the known `--emit-waves` hazard). The proper fix is
|
||||||
|
a separate PR: move the rule and its tests into `platform/keycloak`, then delete the phantom.
|
||||||
|
- **The catalog tag can drift.** Nothing ties `catalog/keycloak.yaml`'s documentation tag to the
|
||||||
|
operator URLs. It must be updated by hand after an operator bump, and the README says so.
|
||||||
|
- **Concurrent branches** `fix/renovate-hold-auth-and-self` and `feat/pin-pihole-jackett-plex`
|
||||||
|
also edit `renovate.json5`. Neither had merged when I checked. A rebase may be needed.
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-31
|
||||||
|
branch: chore/pm-agent-go-live
|
||||||
|
title: Take pm-agent out of dry run (operator-approved); label scope and worker caps unchanged
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- `platform/pm-agent/configmap.yaml`: `PM_DRY_RUN` `"true"` -> `"false"`. The operator approved
|
||||||
|
this after reading the dry-run triage of issue #97. `PM_REQUIRE_LABEL: agent`,
|
||||||
|
`PM_MAX_ACTIVE_WORKERS: 2`, `PM_MAX_NEW_WORKERS_PER_CYCLE: 1`, the worker image digest and every
|
||||||
|
other knob are unchanged. The two comments that said dry run was the only mode that worked
|
||||||
|
(because the run-340 digest lacked `imagePullSecrets`) now describe the run-383 digest that
|
||||||
|
PR #96 pinned.
|
||||||
|
- `platform/pm-agent/deployment.yaml`: `config-generation` `"3"` -> `"4"`. Without the bump the
|
||||||
|
dispatcher keeps its startup config and stays in dry run.
|
||||||
|
- `platform/pm-agent/README.md` and `docs/plans/pm-agent-issue-to-pr.md` (follow-up 3): the
|
||||||
|
"currently dry run" status text now says dry run is off. `runbooks/pm-agent.md` is unchanged.
|
||||||
|
Its "stop it: set `PM_DRY_RUN: "true"`" step and its admission-refusal troubleshooting row
|
||||||
|
are still accurate.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
Read-only, `kubectl --context k3s-prod` through the toolbox, 2026-10-31 ~11:45 local, with Flux
|
||||||
|
`pm-agent` and `platform-config` both at `main@1126740`:
|
||||||
|
|
||||||
|
- ClusterPolicy `pm-agent-job-shape`: `Ready=True`, `validationFailureAction=Enforce`,
|
||||||
|
`background=false`. It has seven rules, including `worker-pull-secret-must-be-the-committed-one`.
|
||||||
|
- RBAC: ServiceAccounts `pm-agent` and `pm-agent-worker`, Role `pm-agent` and RoleBinding
|
||||||
|
`pm-agent` (roleRef `pm-agent`, subject `ServiceAccount/pm-agent`) all exist. `auth can-i`
|
||||||
|
as `system:serviceaccount:pm-agent:pm-agent` returned `yes` for `create jobs.batch` and `no`
|
||||||
|
for `delete jobs.batch`.
|
||||||
|
- Secrets (key names only): `pm-agent-registry` is `kubernetes.io/dockerconfigjson` with key
|
||||||
|
`.dockerconfigjson`. `pm-agent-credentials` is `Opaque` with keys `claude-code-oauth-token`
|
||||||
|
and `gitea-token`.
|
||||||
|
- Egress: `default-deny` NetworkPolicy exists. CiliumNetworkPolicies `pm-agent-allow-egress`
|
||||||
|
and `pm-agent-worker-allow-egress` both report `Valid=True`.
|
||||||
|
- Deployment `pm-agent` is 1/1 on `agent-runner@sha256:df36d9b7…` with
|
||||||
|
`config-generation: "3"` live. No Jobs exist in the namespace.
|
||||||
|
- `platform/pm-agent/{configmap,deployment}.yaml`, README and docs do not match any
|
||||||
|
`sensitive_paths` glob in `infra/gitea/approval-policy.json`, so no review or override applies.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **That a real worker Job is admitted and completes.** Nothing can show this until a labelled
|
||||||
|
issue gets a `start` decision after the merge. The L0 Kyverno suite shows the policy accepts
|
||||||
|
the committed template. Whether the API server admits the Job built from the df36d9b image's
|
||||||
|
baked template has not been tested live.
|
||||||
|
- That the worker pod can pull its image with `pm-agent-registry`, reach Gitea and the model
|
||||||
|
through its egress policy, and push a branch and open a PR. None of this has run before.
|
||||||
|
- The pod list also shows a stale `pm-agent-65db599649-v678g` (25d, `ContainerStatusUnknown`,
|
||||||
|
from an old ReplicaSet). It does not block anything, but I did not investigate it. Clearing
|
||||||
|
it is an operator action.
|
||||||
|
- CI: see the PR for the gate result. Locally the best possible result is rc 3, with only
|
||||||
|
`gitea-settings` not run.
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-31
|
||||||
|
branch: chore/pm-agent-runner-digest
|
||||||
|
title: Bump pm-agent to the run-383 agent-runner digest by hand; Renovate saw it but the schedule withheld it
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
`platform/pm-agent/deployment.yaml` (the dispatcher `image:`) and `platform/pm-agent/configmap.yaml`
|
||||||
|
(`PM_WORKER_IMAGE`) move:
|
||||||
|
|
||||||
|
- from `agent-runner@sha256:c4eba5b3c508bf456a03391ac76f8275e7ea28a829da1450d99f6f6f25c6eb42`, the
|
||||||
|
run-340 publish;
|
||||||
|
- to `agent-runner@sha256:df36d9b7fef56d141220a387f64fb864cede8e5af30513ebfeb3ab566d3f771d`, the
|
||||||
|
images run 383 publish of ddd04d4.
|
||||||
|
|
||||||
|
Both `# renovate-digest:` markers are unchanged, directly above their values.
|
||||||
|
|
||||||
|
`pm-agent.homelab.keithjackson.dev/config-generation` goes `"2"` → `"3"`. That is the convention in
|
||||||
|
`deployment.yaml`'s own comment: "bump it in the same commit as a change to configmap.yaml". The
|
||||||
|
image change alone would roll the pod, but the convention is followed regardless.
|
||||||
|
|
||||||
|
No `*.sops.yaml` file is touched. Branched from 06d4a9a, which already contains the operator's #95.
|
||||||
|
|
||||||
|
### Why Renovate did not do this: the schedule, not a config defect
|
||||||
|
|
||||||
|
- **Run 385** (dispatch, 2026-10-31 16:14 UTC, on ddd04d4) was a **real** run. It logged
|
||||||
|
`Deleting orphan branch (… renovate/pin-dependencies)`, not a `DRY-RUN:` line. It ran at **12:14
|
||||||
|
America/New_York**, outside `"schedule": ["after 3am and before 6am"]`, so it created and updated
|
||||||
|
no branch.
|
||||||
|
- **The lookup DID see the new digest.** Dependency Dashboard issue #91 lists, under **"Awaiting
|
||||||
|
Schedule"**:
|
||||||
|
`chore(deps): Update gitea.home.keithjackson.dev/home-network/agent-runner:main Docker digest to df36d9b`.
|
||||||
|
- **Nothing else blocked it.** There are no open Renovate PRs and no `renovate/*` branch on Gitea
|
||||||
|
(111 branches listed across all pages). The previous agent-runner PR, #90 (to c4eba5b), was
|
||||||
|
auto-merged by renovate-bot at 08:22 on 10-31. No PR limit was hit, and digests are not
|
||||||
|
dashboard-approval gated.
|
||||||
|
- **The registry, read-only, with Docker/OCI `Accept` headers:**
|
||||||
|
- `HEAD …/agent-runner/manifests/main` → `docker-content-digest: sha256:df36d9b7…771d`
|
||||||
|
- `HEAD …/agent-runner/manifests/sha-ddd04d460ef6` → the same digest.
|
||||||
|
|
||||||
|
**Without this PR** the next scheduled run (08:15 UTC, 04:15 EDT) would open it, and it would
|
||||||
|
auto-merge as a `digest` update. Ticking that "Awaiting Schedule" checkbox on #91 makes the next
|
||||||
|
run of any kind create it out of schedule.
|
||||||
|
|
||||||
|
### Found on the way: Renovate's labels have never been applied
|
||||||
|
|
||||||
|
- The repo has exactly one label, `agent`. The org has none.
|
||||||
|
- Every Renovate PR so far (#89, #90, #92, #93) was created with `"labels": []`.
|
||||||
|
- On Gitea, Renovate drops a label that does not exist in the repo. So `needs-review`,
|
||||||
|
`blast-radius`, `range-widen`, `own-image`, `dependencies`, `automated-update`, `breaking` and
|
||||||
|
`ci-tooling` are never attached.
|
||||||
|
- The **automerge flags still work.** Only the labels, the visible review signal, are missing.
|
||||||
|
- **Operator fix:** create those labels on the repo. This was not changed here.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **The new image carries the fix.** I fetched both images' layers read-only from the registry and
|
||||||
|
read `/etc/pm-agent/worker-job-template.yaml` from each (`infra/agent/Containerfile:116` copies
|
||||||
|
it there):
|
||||||
|
- df36d9b's has `imagePullSecrets:` / `- name: pm-agent-registry`;
|
||||||
|
- c4eba5b's has no `imagePullSecrets` at all.
|
||||||
|
|
||||||
|
This matches `infra/agent/worker-job-template.yaml:76-77` at ddd04d4.
|
||||||
|
- **`main` = `sha-ddd04d460ef6` = df36d9b**, both measured.
|
||||||
|
- **No other file references c4eba5b.** No admission policy pins it.
|
||||||
|
- **No override is needed.** `infra/gitea/approval-policy.json` lists only `platform/kyverno/**`
|
||||||
|
under `platform/`, so `platform/pm-agent/**` is not sensitive.
|
||||||
|
- I read the whole job logs of runs 362, 367 and 385.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **That the rolled dispatcher starts on df36d9b and that a worker Job is admitted.** That needs
|
||||||
|
Flux to apply this and a real dispatch. Both are operator actions.
|
||||||
|
- **That the `pm-agent-registry` Secret exists and decrypts in-cluster** (#95's sops file). I did not
|
||||||
|
read it.
|
||||||
|
- **That Renovate will treat this hand bump as current.** It should: its next lookup finds
|
||||||
|
`main` == the pinned digest, so it proposes nothing and drops the dashboard entry. Not observed.
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-31
|
||||||
|
branch: fix/pm-agent-triage-exit
|
||||||
|
title: pm-agent triage fails with "claude exited 1:" and no message; log the CLI's stdout envelope so the cause is visible
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- `infra/agent/pm-dispatch.py`, `ask_pm()`: on a non-zero exit the error now comes from a new
|
||||||
|
`describe_cli_failure()`, not from `proc.stderr` alone. With `--output-format json` the CLI
|
||||||
|
reports its own failures as the result envelope on **stdout** and writes nothing to stderr,
|
||||||
|
which is why both failures today logged `claude exited 1: ` with an empty message. The new
|
||||||
|
line carries the envelope's `subtype`, `is_error`, `api_error_status`, `num_turns`,
|
||||||
|
`duration_ms`, `total_cost_usd`, `stop_reason`, its `result`/`error` text, and the tails of
|
||||||
|
stdout and stderr. It names silence explicitly when both streams are empty.
|
||||||
|
- `redact()`: everything in that message, and in `extract_decision()`'s "no decision object"
|
||||||
|
error, passes through it. It covers `sk-ant-…`, `Bearer`/`token` values, and `user:pass@` in
|
||||||
|
both `http://` and `https://` URLs.
|
||||||
|
- `PM_DECISION_MAX_TURNS`: a new knob. It defaults to 1, today's value, so behaviour is
|
||||||
|
unchanged, and it is bounded to 1–5. If the newly visible envelope says `error_max_turns`,
|
||||||
|
the remedy is a ConfigMap change rather than another image build. Documented in
|
||||||
|
`platform/pm-agent/README.md`.
|
||||||
|
- Selftest: four new assertions. They cover an empty stderr with the cause on stdout, a
|
||||||
|
stderr-only failure, total silence, and redaction.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Timeline (Loki, `{namespace="pm-agent"}`, and `kubectl logs`):**
|
||||||
|
- **16:37:38 to 16:38:00**: pod `pm-agent-898fd8f5f-dp5hs`, image **c4eba5b3**,
|
||||||
|
config-generation 2, dry run. It triaged #97 in **22 s**, logged `PM says comment`, and
|
||||||
|
posted the 16:38:00Z comment. The thread then held only the issue body.
|
||||||
|
- **16:43:09 to 16:44:00**: pod `pm-agent-f7c9b7fcf-j8mzn`, image **df36d9b7**,
|
||||||
|
config-generation 3, **still dry run**. It logged `claude exited 1: ` with an empty
|
||||||
|
message after **51 s**.
|
||||||
|
- **16:55:42 to 16:56:36**: pod `pm-agent-7796db8bcb-p5dsm`, df36d9b7, config-generation
|
||||||
|
4, live. The same empty failure, after **54 s**.
|
||||||
|
- The only successful triage ever was the 16:38 one. Every earlier `898fd8f5f`/`5db55d558c`
|
||||||
|
log line is a startup crash (Gitea `GET /user` 401), not a triage.
|
||||||
|
- **Going live is NOT the cause.** The first failure happened in dry run. `ask_pm()` has no
|
||||||
|
dry-run branch: the prompt, flags, tools (`--tools ""`) and cwd are identical in both
|
||||||
|
modes. `PM_DRY_RUN` gates only `create_job()`, which runs after the model has answered.
|
||||||
|
- **Ruled out, with evidence:**
|
||||||
|
- **Timeout**: 51 to 54 s against a 900 s limit, and a timeout raises `TimeoutExpired`,
|
||||||
|
not rc 1.
|
||||||
|
- **Budget cap**: the PM call passes no `--max-budget-usd`.
|
||||||
|
- **Prompt size**: the thread is the issue body (293 chars) plus three comments
|
||||||
|
(1871 + 404 + 77 chars), and the persona is 8 KB.
|
||||||
|
- **Writable HOME**: HOME=`/workspace` (emptyDir), and `CLAUDE_CONFIG_DIR=/workspace/.claude`
|
||||||
|
was written at 16:56.
|
||||||
|
- **Egress**: a TLS 1.3 handshake to `api.anthropic.com:443` from the dispatcher pod
|
||||||
|
succeeds in 0.06 s.
|
||||||
|
- **Model name and CLI version**: the same `claude-opus-5` and CLI 2.1.285 succeeded at
|
||||||
|
16:38. Between c4eba5b and df36d9b, `infra/agent/` differs only by a template selftest
|
||||||
|
assertion and the toolbox base digest.
|
||||||
|
- **A hint, not proof:** `/workspace/.claude/.claude.json` in the live pod has
|
||||||
|
`cachedExtraUsageDisabledReason: 'out_of_credits'`. No token was read or printed.
|
||||||
|
- Offline selftest: `PM_SELFTEST=1` against the worktree's template and persona prints
|
||||||
|
`selftest OK`, including the 4 new assertions.
|
||||||
|
- Toolbox gate: see the PR description for the rc and the stages that were skipped.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **The root cause. UNVERIFIED.** The CLI's own message was discarded, `--no-session-persistence`
|
||||||
|
leaves no transcript, and I did not run `claude` in the pod. The candidates still open:
|
||||||
|
- **Usage or rate limit on the subscription seat.** The seat is shared with the operator's
|
||||||
|
interactive use. The `out_of_credits` extra-usage flag fits this. So does a ~50 s run,
|
||||||
|
which matches the CLI exhausting its 429/529 retries.
|
||||||
|
- **`error_max_turns`.** With `--max-turns 1` and `--json-schema`, an answer that fails the
|
||||||
|
schema cannot be retried. The failing thread now invites `implement` (a branch plus a
|
||||||
|
long brief), where the 16:38 one produced a short `comment`. ~50 s is about two
|
||||||
|
generations of the 22 s success.
|
||||||
|
- **The 17:11 retry will NOT reveal the cause.** It runs the old image, which still discards
|
||||||
|
stdout. The cause becomes visible only after this PR merges, CI re-publishes agent-runner,
|
||||||
|
and a follow-up PR bumps the dispatcher digest (`deployment.yaml` and the `PM_WORKER_IMAGE`
|
||||||
|
in `configmap.yaml`, plus a `config-generation` bump).
|
||||||
|
- That the CLI's error envelope carries exactly these key names. The parser takes whichever
|
||||||
|
keys are present and falls back to raw tails.
|
||||||
@@ -0,0 +1,441 @@
|
|||||||
|
---
|
||||||
|
date: 2026-10-31
|
||||||
|
branch: docs/source-comment-audit
|
||||||
|
title: Audit — historical narrative in source comments (read-only inventory, sample not exhaustive)
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
**Nothing but this file.** This is the read-only audit asked for in issue #97. No source
|
||||||
|
file was edited; the pruning is to be briefed afterwards, per directory, from this list.
|
||||||
|
|
||||||
|
### What was asked, and the calibration applied
|
||||||
|
|
||||||
|
From the reporter (issue #97, 2026-10-31T16:42:30Z): *"'historical' talks about previous
|
||||||
|
issues or intent that have changed and why/how it is used today within comments of source
|
||||||
|
files rather than keeping them in the decisions and documentation. Source file comments
|
||||||
|
should only talk about the current files and sections of code/yaml purposes, uses,
|
||||||
|
expectations."*
|
||||||
|
|
||||||
|
Each row below is classified:
|
||||||
|
|
||||||
|
- **`historical`** — narrative about a superseded state, a past incident, a corrected
|
||||||
|
earlier belief, or a closed issue. Says nothing about what the line does today.
|
||||||
|
- **`load-bearing`** — phrased as history, but the sentence is the only record of a
|
||||||
|
**current** non-obvious constraint. Per the brief these get **rewrite**, never delete.
|
||||||
|
- **`ambiguous`** — I could not rule without the operator. Left for a human, deliberately
|
||||||
|
not resolved here.
|
||||||
|
|
||||||
|
### THIS IS A SAMPLE, NOT AN EXHAUSTIVE INVENTORY
|
||||||
|
|
||||||
|
The brief caps the audit at roughly 40–50 findings and asks for honesty over padding.
|
||||||
|
Measured candidate volume, tree-wide, with the keyword sweep below:
|
||||||
|
|
||||||
|
```
|
||||||
|
git ls-files -z | xargs -0 grep -nEi '#.*(used to|previously|formerly|no longer|we had|
|
||||||
|
before we|historically|in the past|originally|superseded|turned out|was wrong|predates|
|
||||||
|
legacy|issue #[0-9]+|PR #[0-9]+|measured 20|as of 20[0-9][0-9]-|see decisions/|DECISIONS-LOG)'
|
||||||
|
```
|
||||||
|
|
||||||
|
| Directory | candidate hits | files | coverage here | findings listed |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `platform/` | 107 | 56 | **sampled** | 9 |
|
||||||
|
| `infra/` | 75 | 44 | **sampled** | 8 |
|
||||||
|
| `conformance/` (excl. `runs/`) | 72 | ~31 | **sampled** | 5 |
|
||||||
|
| `clusters/` | 43 | 19 | **sampled** | 7 |
|
||||||
|
| `ci/` | 42 | 15 | **sampled** | 6 |
|
||||||
|
| `catalog/` | 39 | 9 | **sampled** | 7 |
|
||||||
|
| `.gitea/` | 10 | 5 | **full** | 5 |
|
||||||
|
| `bootstrap/` | 4 | 3 | **full** | 3 |
|
||||||
|
| `platform-config/` | 3 | 1 | **full** | 3 |
|
||||||
|
| `apps/vms/` (hand-authored) | 3 | 2 | **full** | 2 |
|
||||||
|
| `apps/wave-*` (generated) | 15 | — | **excluded — see below** | 0 |
|
||||||
|
| `conformance/runs/` (archived logs) | 8 | 18 files | **excluded — see below** | 0 |
|
||||||
|
| `runbooks/` | 11 | — | **not audited** (prose docs, not source comments) | 0 |
|
||||||
|
| **total** | **~420** | **~190** | — | **55** |
|
||||||
|
|
||||||
|
Covered **fully**: `.gitea/`, `bootstrap/`, `platform-config/`, `apps/vms/`.
|
||||||
|
Covered **by sample only**: `platform/`, `infra/`, `conformance/`, `clusters/`, `ci/`,
|
||||||
|
`catalog/`. Roughly **365 candidate hits remain unexamined**, and candidate ≠ finding —
|
||||||
|
the keyword sweep's precision on the parts I did read was well under half, so the true
|
||||||
|
residual is smaller than 365 but is certainly in the low hundreds.
|
||||||
|
|
||||||
|
**The issue's premise holds**, with one qualification the operator should price in: of the
|
||||||
|
55 rows below, **22 are `load-bearing`** and get a rewrite rather than a deletion. The
|
||||||
|
narrative is genuinely excessive, but a mechanical "delete the history" sweep would destroy
|
||||||
|
real constraints. That ratio is the single most important number in this report.
|
||||||
|
|
||||||
|
### Two structural warnings that change how the follow-up PRs must be written
|
||||||
|
|
||||||
|
**1. `apps/wave-0/`, `apps/wave-1/`, `apps/wave-2/` are GENERATED. Do not edit comments there.**
|
||||||
|
They are goldens produced by one canonical invocation of `catalog/render.py`
|
||||||
|
(`ci/render-golden.sh:21`), and `ci/render-golden.sh:136-150` byte-compares every rendered
|
||||||
|
file against its committed golden — **any diff is a hard `exit 1` that blocks the merge**.
|
||||||
|
So the 15 candidate hits under `apps/wave-*` are *copies* of comments whose source is
|
||||||
|
`catalog/*.yaml` or `catalog/render.py`; e.g. `apps/wave-0/gitea.yaml:130` is the rendered
|
||||||
|
form of `catalog/gitea.yaml:100`. Edit the catalog source, then re-render with the canonical
|
||||||
|
command and commit both. A pruning PR that hand-edits `apps/wave-*` will sit red.
|
||||||
|
|
||||||
|
**2. `conformance/runs/*/run.log` are archived evidence artifacts, not source.** 8 candidate
|
||||||
|
hits sit in 18 committed run logs. They are the recorded output of past conformance runs —
|
||||||
|
the comments inside them are copies of the test bodies as they stood at run time. Rewriting
|
||||||
|
them would falsify evidence. Excluded from scope entirely; recommend the operator confirm.
|
||||||
|
|
||||||
|
### A dangling pointer, replicated across 18 files — fix this before any "replace with a pointer" work
|
||||||
|
|
||||||
|
**18 files cite `DECISIONS-LOG: "Headlamp gating stall"` as their rationale pointer, and no
|
||||||
|
such entry exists.** `grep -ci "gating stall" DECISIONS-LOG.md` → **0**; the phrase appears
|
||||||
|
nowhere in `decisions/` either. Citers:
|
||||||
|
|
||||||
|
```
|
||||||
|
clusters/base/crafty.yaml clusters/base/headlamp.yaml
|
||||||
|
clusters/base/homepage.yaml clusters/base/kustomization.yaml
|
||||||
|
clusters/base/runbooks.yaml clusters/base/vpn-ingress.yaml
|
||||||
|
platform-config/kustomization.yaml platform/crafty/deployment.yaml
|
||||||
|
platform/crafty/kustomization.yaml platform/headlamp/kustomization.yaml
|
||||||
|
platform/homepage/kustomization.yaml platform/homepage/README.md
|
||||||
|
runbooks/kustomization.yaml runbooks/flux-reconcile.md
|
||||||
|
conformance/tests/l0-static/alert-runbooks-resolvable/check.sh (+3 more)
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the audit's clearest warning about its own remedy: "replace the narrative with a
|
||||||
|
pointer" is only an improvement if the target exists and keeps existing. The archive is
|
||||||
|
frozen and has no index, so a title-based citation into it cannot be checked by anything.
|
||||||
|
**Recommendation:** when the operator takes the pruning PRs, either write a real
|
||||||
|
`decisions/` entry for the leaf-gating lesson and repoint all 18 citations at its filename,
|
||||||
|
or drop the citation. Pointer targets of the form `decisions/<file>.md` are checkable;
|
||||||
|
`DECISIONS-LOG: "<title>"` is not. Nothing below recommends a pointer target I did not
|
||||||
|
confirm exists.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `platform/` — 107 candidates, 9 listed (sampled). NOT sensitive, except `platform/kyverno/**`
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 1 | `platform/crafty/service-l4.yaml:33-91` | "CORRECTED 2026-09-14 after an 8-DAY LAN OUTAGE … This file used to say `Local`" — 58 lines of outage narrative | load-bearing | **rewrite.** Keep the current constraints (Cilium's L2 announcer is not backend-aware; SNAT mode loses client IP; `/ban-ip` degrades so use `/ban`; `remote-node` is subsumed by `cluster` so do NOT add it; no `loadBalancerSourceRanges` deliberately). Drop the outage story, the lease-age detail and the per-protocol tell | no |
|
||||||
|
| 2 | `platform/longhorn/helmrelease.yaml:191-200` | "`orphanAutoDeletion: true` USED TO BE HERE AND WAS INERT. Removed 2026-09-13" | load-bearing | **rewrite** to the forward-looking rule: Longhorn ≥1.9 replaced it with `orphanResourceAutoDeletion` and the chart silently drops unknown `defaultSettings` keys — so do not set it | no |
|
||||||
|
| 3 | `platform/longhorn/helmrelease.yaml:249-254` | "CORRECTION, re-measured 2026-09-13 later the same day: the line originally recorded here read …" — corrects a superseded measurement | historical | **delete.** The correct reading is already stated at `:246-248` | no |
|
||||||
|
| 4 | `platform/monitoring/alerts.yaml:625-636` | "Both rules below used to read `== 2` / `== 3`, with the comment 0=unknown 1=healthy…" | load-bearing | **rewrite.** Keep "`longhorn_volume_robustness` is a one-hot `state`-labelled gauge, value 0 or 1" — that is why the expressions look as they do. Drop the old-enum story | no |
|
||||||
|
| 5 | `platform/monitoring/alerts.yaml:869-875` | "This alert originally used `severity: info`. That was WRONG, and wrong in the silent direction" | load-bearing | **rewrite** to the current rule: carries no `severity` deliberately, because `general.rules`' `InfoInhibitor` would swallow an info-severity deploy notification | no |
|
||||||
|
| 6 | `platform/monitoring/pvc-capacity-alerts.yaml:12-27` | "`LonghornVolumeCapacityHigh` … never did, for two different reasons in sequence, and it is now GONE" — dated 2026-09-06/2026-09-13 timeline | load-bearing | **rewrite.** Keep only "these rules key on `kubelet_volume_stats_*` because `longhorn_volume_actual_size_bytes` measures allocation, not filesystem use". Delete the sequence | no |
|
||||||
|
| 7 | `platform/headlamp/helmrelease.yaml:77-85` | "These two used to be keys INSIDE headlamp-oidc-credentials.sops.yaml. That was the bug…" + the dev-Keycloak `i/o timeout` story | load-bearing | **rewrite** to the current rule: issuer and callback URLs are public and MUST be cleartext, because Flux's `${BASE_DOMAIN}` postBuild substitution cannot reach inside an encrypted Secret. Keep the `[parity]` note at `:87-92` as-is | no |
|
||||||
|
| 8 | `platform/vpn-egress/egress-steering.yaml:79` | "CORRECTION 2026-09-12 — this comment used to say the host nftables kill-switch 'is the…'" | load-bearing | **rewrite** to state the actual mechanism (Cilium's eBPF egress-gateway drop) without the correction framing | no |
|
||||||
|
| 9 | `platform/keycloak/passkey-reconciler.yaml:18` | "It used to be duplicated here almost verbatim" | historical | **delete**, or replace with pointer `decisions/2026-10-18-keycloak-passkey-reconciler.md` (confirmed present) | no |
|
||||||
|
|
||||||
|
Also seen in `platform/` and left unlisted for the cap, same shapes: `platform/cilium/helmrelease.yaml:57-65`
|
||||||
|
(load-bearing — the `k3s_api_host`/`k3s_apiserver_endpoint` split is the current parity contract),
|
||||||
|
`platform/kubevirt/kubevirt-cr.yaml:7` (`useEmulation was previously true`), `platform/external-snapshotter/kustomization.yaml:35`
|
||||||
|
(`Previously these were REMOTE`), `platform/kustomization.yaml:7` (`the same pattern by which kyverno was previously deferred`),
|
||||||
|
`platform/pm-agent/configmap.yaml:22`, `platform/monitoring/tests/run.sh:61` and `:129`,
|
||||||
|
`platform/cnpg/alerts.promtool-test.yaml:20`, `platform/oauth2-proxy/auth-visibility-alerts.yaml:36-42`
|
||||||
|
and `:98-101`, `platform/vpn-ingress/daemonset.yaml:165`.
|
||||||
|
|
||||||
|
### `infra/` — 75 candidates, 8 listed (sampled). NOT sensitive, except `infra/gitea/**`
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 10 | `infra/plex/claim-ship.sh:29-32` | "Branch prefix is now cosmetic. It used to matter: gate.yml's push trigger covered feature/** … Those per-branch push triggers were deleted" | historical | **delete.** The live trigger set is already documented at `.gitea/workflows/gate.yml:48-56` and `CLAUDE.md` §4 | no |
|
||||||
|
| 11 | `infra/ansible/inventory/hosts.ini:105` | "GROUP-NAME GAP: CLOSED 2026-08-31. The roles and playbooks used to hardcode the…" | historical | **delete** — the gap is closed and the current variable is named at `playbooks/node-health.yml:12` | no |
|
||||||
|
| 12 | `infra/ansible/inventory/hosts.ini:147-153` | "NOTE (CORRECTED 2026-09-12): this used to read 'node 3 also has wlp1s0 up on 192.168.2.195 (wifi)'. It does not" | load-bearing | **rewrite** as a positive present-tense assertion: node 3 has no `wl*` netdev and a single IPv4 uplink on `eno1`. That fact is load-bearing for the kill-switch datapath; only the correction framing should go | no |
|
||||||
|
| 13 | `infra/ansible/roles/node_health/defaults/main.yml:110-121` | "The first real run of this role FAILED k3s-prod-1's Crucial/Micron C400 on `reallocated=61440` … That verdict was WRONG" | load-bearing | **rewrite.** Keep the current rule (SMART raw values are vendor-defined; Micron TN-FD-20 scales ID 5 by `Block_Sector_Count`, so ID 170/196 are the trustworthy corroborators). Drop "the first real run failed" | no |
|
||||||
|
| 14 | `infra/ansible/roles/unattended_upgrades/tasks/main.yml:304-309` | "This was originally a `find` over /etc/kernel/postinst.d … That theory was wrong" | load-bearing | **rewrite** to the current justification: the sentinel writer is ours at two known paths, and two exact `stat`s are strictly stronger than a glob that would match Debian's own conffile | no |
|
||||||
|
| 15 | `infra/ansible/roles/node_health/tasks/assess.yml:398` | "measurement was right; the inference from the log line was wrong — the memattr block below…" | historical | **delete** | no |
|
||||||
|
| 16 | `infra/devcontainer/dev.sh:103-108` | "SUPERSEDED but still honoured: `DEV_KUBECONFIG` used to be the only way to reach a cluster other…" | load-bearing | **rewrite.** "Still honoured" is current behaviour and `CLAUDE.md` §3 tells every agent to set `DEV_KUBECONFIG=/dev/null`, so this must keep documenting the variable — just not as history | no |
|
||||||
|
| 17 | `infra/vpn-ingress/new-peer.sh:136` | "COMMENT LINES ARE NOT ALLOCATIONS. This used to grep the whole file and so matched the…" | load-bearing | **rewrite**, replacing the history with pointer `decisions/2026-10-06-new-peer-ignores-template-comments.md` (confirmed present) | no |
|
||||||
|
|
||||||
|
Unlisted, same shapes: `infra/ansible/inventory/group_vars/all.yml:31`,
|
||||||
|
`infra/ansible/inventory/group_vars/k3s_prod/vars.yml:86` ("CORRECTION 2026-09-06: an earlier
|
||||||
|
version of this comment claimed…"), `infra/ansible/roles/node_baseline/tasks/disks.yml:559`,
|
||||||
|
`infra/ansible/roles/node_baseline/tasks/main.yml:801`, `infra/ansible/roles/kube_vip/tasks/main.yml:68`,
|
||||||
|
`infra/ansible/roles/cilium/defaults/main.yml:30`, `infra/ansible/playbooks/node-health.yml:12`,
|
||||||
|
`infra/ansible/roles/vpn_egress_gateway/templates/vpn-egress-up.sh.j2:39` and
|
||||||
|
`vpn-egress-down.sh.j2:15`, `infra/devcontainer/kubeconfig.sh:26`, `infra/agent/Containerfile:216`,
|
||||||
|
`infra/agent/build-image.sh:30`, `infra/agent/pm-worker:97`, `infra/agent/probe-validation.sh:98`.
|
||||||
|
|
||||||
|
### `catalog/` — 39 candidates, 7 listed (sampled). NOT sensitive — but every edit here re-renders `apps/wave-*`
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 18 | `catalog/qbittorrent.yaml:7-12` | "BULK DOWNLOADS TIER — DELIVERED 2026-09-12 (the deferral below is CLOSED; kept for the record). This entry used to ship config-only, with a comment promising … 'once OQ-2 is resolved'" | historical | **delete.** Self-describes as "kept for the record", which is precisely the issue's complaint | no |
|
||||||
|
| 19 | `catalog/qbittorrent.yaml:141-151` | "It USED to be a doubled-dollar default-operator expansion of a WEBUI_PORT env var … That MISDIAGNOSIS is preserved in this service's own marker log at 2026-09-13T05:35Z" | load-bearing | **rewrite.** Keep "PORT is a literal matching `ingressRoutes[0].port`; a literal cannot be mangled by envsubst". Drop the misdiagnosis story (it is already preserved in the marker log the comment itself cites) | no |
|
||||||
|
| 20 | `catalog/plex.yaml:43-46` | "(This comment previously asserted '~4 minutes' as established fact. It was … Measured 2026-09-29: the full loop is ~2m45s and fits.)" | historical | **rewrite** to the measured figure alone; delete the self-correction parenthesis | no |
|
||||||
|
| 21 | `catalog/plex.yaml:65` | "So the earlier 'a placeholder claim Secret would crashloop the pod' note was WRONG about the…" | historical | **delete** | no |
|
||||||
|
| 22 | `catalog/plex.yaml:318` | "actually granted access (device-cgroup EPERM, measured 2026-10-01), so there was nothing to keep" | historical | **rewrite** with pointer `decisions/2026-10-01-quicksync-via-device-plugin.md` and/or `decisions/2026-10-03-cilium-cgroup-hostroot.md` (both confirmed present) | no |
|
||||||
|
| 23 | `catalog/gitea.yaml:14` | "Headless-install bootstrap (formerly `apps/gitea-bootstrap-patch.yaml`; now first-class via the…)" | historical | **delete** the "formerly" clause; keep the description of what the block does | no |
|
||||||
|
| 24 | `catalog/render.py:159-166` | "THE SCRAPE WORKS AS OF 2026-10-05 — but it took THREE independent pieces … (PR #28) … (PR #49)" | load-bearing | **rewrite.** The three-part requirement and "do not read this flag alone as 'metrics are working'" are current and genuinely non-obvious; drop the PR numbers and the "63.7h" incident, pointing at `decisions/2026-10-05-oauth2-sidecar-scrape-verified.md` (confirmed present) | no |
|
||||||
|
|
||||||
|
`catalog/pihole.yaml:210` and `:218` are the **exemplar to copy**: narrative already reduced to
|
||||||
|
`see decisions/2026-10-04-pihole-multi-replica.md` / `-pihole-declare-state.md`, both of which
|
||||||
|
exist. Unlisted: `catalog/gitea.yaml:100-130`, `catalog/nextcloud.yaml:211` and `:226-240`,
|
||||||
|
`catalog/semaphore.yaml:84`, `catalog/pihole.yaml:179`, `catalog/qbittorrent.yaml:389-417`,
|
||||||
|
`catalog/sonarr.yaml:120`, `catalog/render.py:672`, `:1371`, `:2161`.
|
||||||
|
`catalog/jackett.yaml:73` ("floor only, no ceiling; 0.24.x runs as of 2026-10-30") is
|
||||||
|
**load-bearing, leave alone** — a dated version constraint, exactly the carve-out the brief names.
|
||||||
|
|
||||||
|
### `.gitea/` — 10 candidates, 5 listed (FULL coverage). **SENSITIVE** (`.gitea/**`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 25 | `.gitea/workflows/gate.yml:14-16` | "(it used to print '— not green' and then 'MERGE GATE: PASS', exit 0 — fail-open in the gate itself)" | historical | **delete** the parenthesis; the surrounding `GATE_STRICT_SKIPS` rationale is current and stays | **yes** |
|
||||||
|
| 26 | `.gitea/workflows/gate.yml:48-56` | "There is NO per-branch push trigger. There used to be a prefix list (feature/** fix/** …) and it was a net negative here" + the 2026-09-29 plex-claim cost | load-bearing | **rewrite.** "Do not re-add a push trigger; `act_runner` is capacity 1 and the duplicate run serialises ahead of the PR run" is a live constraint a future editor needs. Drop the plex-claim anecdote | **yes** |
|
||||||
|
| 27 | `.gitea/workflows/gate.yml:118-123` | "WIDENED TOLERANCE, DECLARED: `\|\| [ "$?" -eq 3 ]` used to cover exactly one skip (spec-immutability)" | load-bearing | **rewrite.** This documents a deliberate *current* weakening plus its exit condition (`REQUIRE_GITEA_SETTINGS=true` once the secret is seen green). Keep the instruction, drop "used to" | **yes** |
|
||||||
|
| 28 | `.gitea/workflows/renovate.yaml:72-76` | "CORRECTED 2026-10-30. This list used to name only write:repository + read:user … Run 350 then failed at `GET /api/v1/orgs/… 403`" | load-bearing | **rewrite** to the required scope list as a plain present-tense requirement; point at `decisions/2026-10-30-renovate-auth.md` (confirmed present) for the run-350 story | **yes** |
|
||||||
|
| 29 | `.gitea/workflows/merge-gate.yaml:15` | "The text that used to sit here…" | historical | **delete** | **yes** |
|
||||||
|
|
||||||
|
Remaining `.gitea/` candidates examined and judged **not findings** (current-state
|
||||||
|
measurements, correctly phrased): `renovate.yaml:6`, `:34`, `:259`, `images.yaml:44`,
|
||||||
|
`scripts/images.sh:11`.
|
||||||
|
|
||||||
|
### `ci/` — 42 candidates, 6 listed (sampled). **SENSITIVE** (`ci/**`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 30 | `ci/decisions-log.sh:5-23` | 19-line "WHY THIS STAGE EXISTS" block reproducing the four-PR conflict table verbatim | historical | **replace with a pointer.** It is a near-verbatim duplicate of `decisions/README.md:9-30`, which the comment itself already cites at `:23`. Cut to the assertion list + that pointer (and `decisions/2026-09-29-decisions-log-conflict-free.md`, confirmed present) | **yes** |
|
||||||
|
| 31 | `ci/gitea-settings.sh:7-23` | "On 2026-09-29 that blind spot was measured … PR #1 … sat `pending` for 30.6 minutes … PRs #5-#9 merged with zero approvals" | load-bearing | **rewrite.** "No other check can see Gitea's own database; assert-scripts do not detect drift from their own target" is the current reason the stage exists. Drop the PR-by-PR history | **yes** |
|
||||||
|
| 32 | `ci/l0.sh:15-23` | "That union replaces what used to be a hand-written 'UNCONDITIONAL' stage per whole-repo check (3c subst-safety / 3d keycloak-client-secrets / 3e bulk-tier)" | historical | **rewrite** to describe the present selection rule only (touched-requirement subset ∪ every test declaring `scope: whole-repo`) | **yes** |
|
||||||
|
| 33 | `ci/lib.sh:138-142` | "WHY THIS EXISTS: `ci/l0.sh` used to carry a hand-written stage per whole-repo check, each with its own `summary …` line" | historical | **rewrite** to "names every offline check that ran and its verdict" | **yes** |
|
||||||
|
| 34 | `ci/approval-policy.sh:107-112` | "Blocking on that turned 'CI cannot read your override' into a message that said 'you did not post one', and cost a real debugging session on PR #38" | load-bearing | **rewrite.** The token-scope fact and the fail-direction guarantee ("an unreadable source can only cause a false BLOCK, never a false pass") are current; the debugging-session cost is not | **yes** |
|
||||||
|
| 35 | `ci/touched-requirements.sh:288-293` | "R-TEST-06 added 2026-09-12. The comment in `changed_diff()`'s rule 3 above records why it was ABSENT: R-TEST-06 used to be enforced by the spec-immutability STAGE" | load-bearing | **rewrite.** Keep why a `ci/` change must attribute R-TEST-06 today; drop the absence history | **yes** |
|
||||||
|
|
||||||
|
Unlisted: `ci/approval-policy.sh:30-38` and `:182`, `ci/conformance.sh:73`, `ci/gate.sh:133`,
|
||||||
|
`:233-234`, `ci/l0.sh:86`, `ci/l1.sh:33`, `ci/lib.sh:65`, `ci/render-golden.sh:5-6` and `:24`,
|
||||||
|
`ci/touched-requirements.sh:39`, `:316`, plus `ci/tests/*.test.sh` (5 files — note these are
|
||||||
|
test *rationale* headers, where a "this used to fail on main" line arguably earns its keep;
|
||||||
|
I would send the whole `ci/tests/` group to `ambiguous`).
|
||||||
|
|
||||||
|
### `clusters/` — 43 candidates, 7 listed (sampled). **SENSITIVE** (`clusters/**`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 36 | `clusters/base/cluster-vars.yaml:244-251` | "IT IS NO LONGER A CROSS-NETWORK KNOB (2026-10-07). This used to read 'the two clusters are on OPPOSITE SIDES of pfSense'…" | load-bearing | **rewrite.** Keep "exactly one storage host, 192.168.2.30; still a knob only because the renderer reads it per-environment" and the immutability warning at `:253+`. Replace the pfSense/DNAT history with pointer `decisions/2026-10-07-bulk-nfs-truenas.md` (confirmed present) | **yes** |
|
||||||
|
| 37 | `clusters/prod/kustomization.yaml:335-351` | "APEX AS OF 2026-09-13 … Prerequisites completed and VERIFIED before this flip, in this order: 1. dev's 18 A records released … 2. dev's 18 LEGACY UNPREFIXED TXT registry entries deleted by hand … 3. dev's Certificate deleted" | historical | **rewrite.** The three completed prerequisites are done and unrepeatable. Keep only the still-true `NOTE` at `:348-351` (the zone's two wildcards point at the legacy stack's public IP and prod's specific records shadow them) | **yes** |
|
||||||
|
| 38 | `clusters/prod/kustomization.yaml:472-479` | "*** DECIDED 2026-09-06 by the operator: the VIP, as intended. *** (This block previously read 'Not yet decided' while shipping .10 …)" | load-bearing | **rewrite.** Keep "this is the control-plane VIP, NOT .20 the ingress VIP and NOT .10 the single node address" and why a node-pinned CNI endpoint fails silently. Drop the previous-wording parenthesis | **yes** |
|
||||||
|
| 39 | `clusters/base/apps-vms.yaml:51-55` | "postBuild.substituteFrom ENABLED 2026-08-31 (R-ENV-02). It was previously omitted because Flux's envsubst runs over the ENTIRE manifest…" | load-bearing | **rewrite.** The envsubst hazard over VM cloud-init and the `$${...}` escape obligation are live constraints for anyone adding a VM manifest; the enablement date and prior omission are not | **yes** |
|
||||||
|
| 40 | `clusters/dev/kustomization.yaml:282-287` | "THIS VALUE IS INERT — dev is decommissioned, its nodes no longer exist, and the Proxmox host this used to name (192.168.1.9) went with the pools it held" | load-bearing | **keep, trim.** "Inert, restated per this overlay's supply-every-knob convention, unproven because there is no dev node" is honest current state and the kind of comment the issue wants | **yes** |
|
||||||
|
| 41 | `clusters/base/apps-wave-0.yaml:4`, `apps-wave-1.yaml:4`, `apps-wave-2.yaml:4` | "wait:true health-gates ONLY this wave's resources, so a later wave can no longer stall an earlier one" (×3, and the same sentence is emitted by `catalog/render.py:2385`) | load-bearing | **rewrite** the "no longer" to plain present tense. Note the renderer emits this string, so fix `render.py:2385` and re-render rather than editing the three files by hand | **yes** |
|
||||||
|
| 42 | `clusters/base/headlamp.yaml:9-12`, `homepage.yaml:10`, `crafty.yaml:11`, `kustomization.yaml:53`, `runbooks.yaml:15`, `vpn-ingress.yaml:9` | six files citing `DECISIONS-LOG: "Headlamp gating stall"` — **target does not exist** (see above) | historical | **rewrite** to the present-tense rule (a leaf UI belongs in its own post-`platform-config` Kustomization so its crashloop cannot stall the chain) and drop or repoint the dead citation | **yes** |
|
||||||
|
|
||||||
|
Unlisted: `clusters/base/cilium.yaml:52`, `cluster-vars.yaml:29`, `:51`, `:264`,
|
||||||
|
`platform.yaml:17`, `:56`, `platform-config.yaml:6`, `platform-config-cdi.yaml:13`,
|
||||||
|
`oauth2-proxy.yaml:10`, `clusters/dev/kustomization.yaml:67`, `:81`, `:222`,
|
||||||
|
`clusters/dev/sync.yaml:8`, `clusters/prod/kustomization.yaml:277-283`, `:317`, `:363`, `:546`, `:554`.
|
||||||
|
|
||||||
|
### `conformance/` — 72 candidates (excl. `runs/`), 5 listed (sampled). **SENSITIVE** (`conformance/**`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 43 | `conformance/tests/l3-fault-injection/vpn-killswitch/chainsaw-test.yaml:16-24` | "MECHANISM CORRECTED 2026-09-12: this comment used to credit 'the host nftables kill-switch' … The ASSERTION below is unchanged and still the right one — only the explanation was wrong" | load-bearing | **rewrite** to state the real mechanism (Cilium's eBPF egress-gateway drop, netfilter never invoked) in the present tense. Pairs with finding #8 — same correction duplicated in `platform/vpn-egress/` | **yes** |
|
||||||
|
| 44 | `conformance/tests/l3-fault-injection/vm-data-survival/chainsaw-test.yaml:18-27` | "(Feature-gate note, revalidated 2026-08: an earlier revision of this comment also claimed the containerDisk BOOT path was broken … That premise is now STALE: the cluster is k3s v1.36.4, where ImageVolume is ENABLED)" | historical | **rewrite** to the one current sentence that matters — DataVolume is used because only it survives a rebuild — and drop the whole stale-premise parenthesis | **yes** |
|
||||||
|
| 45 | `conformance/tests/l0-static/ha-control-plane-as-code/chainsaw-test.yaml:51-60` | "TWO GAPS CLOSED 2026-09-06 (they used to be listed above; recorded so the change is legible)" | historical | **rewrite.** Both gaps are closed and now asserted; state the assertions, drop the gap history. Same in `check.sh:98` and `:500` ("This section used to be an ESCALATION in the header's gap list") | **yes** |
|
||||||
|
| 46 | `conformance/lib/status.py:303-310` | "It used to be, and that mis-reported a SAFETY-CRITICAL requirement … A previous engineer worked around it by rewording prose" | load-bearing | **rewrite.** "Scaffold detection must name the idioms it hunts for, because a placeholder-detection test legitimately contains the words" is the current design constraint; the R-STATE-03 misreport and the previous engineer are not | **yes** |
|
||||||
|
| 47 | `conformance/tests/l0-static/bulk-tier-distinct-from-replicated/check.sh:447-455` | "It used to assert `… in L0`: ci/l0.sh carried a hand-written 'UNCONDITIONAL' stage per whole-repo check" — **the same paragraph is duplicated verbatim in 3 files** (also `flux-postbuild-substitution/check.sh:604`, `keycloak-client-secret-reconciler/check.sh:678`) | historical | **rewrite** all three to the present mechanism (the test declares `scope: whole-repo`; `ci/l0.sh` unions declaring tests via `--include-scope`; both halves checked). Good candidate for one small PR — identical edit ×3 | **yes** |
|
||||||
|
|
||||||
|
Unlisted: `conformance/incluster/chainsaw-job.yaml:44`, `conformance/incluster/rbac.yaml:105`
|
||||||
|
(**also sensitive as `**/rbac.yaml`** — "superseded design" for the gluetun Deployment;
|
||||||
|
load-bearing, since it explains why the fault Job needs no RBAC), `conformance/lib/common.sh:33`,
|
||||||
|
`conformance/lib/status.py:114`, and ~10 further `tests/l0-static/*/check.sh` headers of the
|
||||||
|
same shape (`alert-metrics-have-scrape-targets`, `alert-rules-unit-tested`,
|
||||||
|
`alert-runbooks-resolvable`, `auditability-proofs-producible`, `keycloak-passkey-reconciler`).
|
||||||
|
|
||||||
|
### `bootstrap/` — 4 candidates, 3 listed (FULL coverage). **SENSITIVE** (`bootstrap/**`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 48 | `bootstrap/flux/components/kustomization.yaml:36-38` | "On 2026-09-01 that shipped `k8sServiceHost: ""` into the Cilium HelmRelease and put 2 of 3 agents into Init:Error (DECISIONS-LOG, INCIDENT entry)" | load-bearing | **rewrite.** The fail-open default and what the feature gate changes are the current justification; the incident is one clause. Pointer is vague ("INCIDENT entry") — `grep` finds 2026-09-01 material in the archive, so if a citation is kept it should name a locatable heading, not "INCIDENT entry" | **yes** |
|
||||||
|
| 49 | `bootstrap/flux/bootstrap.sh:84-86` | "see ./components/kustomization.yaml and the 2026-09-01 INCIDENT entry in DECISIONS-LOG.md" | historical | **rewrite** to keep only the cross-reference to `./components/kustomization.yaml`, which is checkable; drop the archive citation (duplicate of #48) | **yes** |
|
||||||
|
| 50 | `bootstrap/sops/secret-templates/pia-custom-ovpn.yaml.example:6` | `See DECISIONS-LOG "Phase 3 VPN — custom .ovpn"` | ambiguous | **operator call.** The target *does* exist (10 hits for "Phase 3 VPN" in the archive), so this is a working pointer of the discouraged form. Keep, or repoint if a `decisions/` entry is written. Cited by path+line only; no content quoted (secret-adjacent `.example` template) | no (`*.sops.yaml.example` is not matched by `**/*.sops.yaml`) |
|
||||||
|
|
||||||
|
Remaining candidate `bootstrap/flux/bootstrap.sh:122` ("a patch target that no longer exists
|
||||||
|
after a Flux…") examined — **not a finding**, it describes current failure behaviour.
|
||||||
|
|
||||||
|
### `platform-config/` — 3 candidates, 3 listed (FULL coverage). **SENSITIVE** (`platform-config/kustomization.yaml`)
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 51 | `platform-config/kustomization.yaml:8` | "See `clusters/base/platform-config.yaml` + DECISIONS-LOG." | historical | **rewrite** to keep the checkable half (`clusters/base/platform-config.yaml`) and drop the bare, untargeted `DECISIONS-LOG` reference — it names no entry and cannot be resolved | **yes** |
|
||||||
|
| 52 | `platform-config/kustomization.yaml:93` | "Invisible on dev, whose CDI CRDs have existed for months. See DECISIONS-LOG 2026-09-05." | load-bearing | **rewrite.** "THE RULE" at `:95-96` is the keeper; the dev-invisibility observation is stale (dev is decommissioned — see finding #40) and should go | **yes** |
|
||||||
|
| 53 | `platform-config/kustomization.yaml:167` | "headlamp: httproute MOVED OUT … Keeping it here … meant a headlamp crashloop stalled platform-config. See DECISIONS-LOG 'Headlamp gating stall'." | historical | **rewrite** to the present-tense statement of where the HTTPRoute lives now; **dead pointer**, see the dangling-pointer section | **yes** |
|
||||||
|
|
||||||
|
### `apps/vms/` — 3 candidates, 2 listed (FULL coverage). Hand-authored, NOT generated
|
||||||
|
|
||||||
|
| # | path:lines | comment (gist) | class | action | sensitive |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| 54 | `apps/vms/unifi/50-service-l4.yaml:27-78` | "CORRECTED 2026-09-14 after an 8-DAY LAN OUTAGE … This file used to say `Local` … BOTH HALVES OF THAT WERE FALSE" — ~52 lines | load-bearing | **rewrite**, same treatment as #1 (its near-twin). Keep: L2 announcer is not backend-aware; KubeVirt `masquerade` already destroyed the client IP so `Cluster` costs nothing here; `cluster` subsumes `remote-node` so do NOT "fix" the CNP; no `loadBalancerSourceRanges` deliberately. Drop the outage story | no |
|
||||||
|
| 55 | `apps/vms/unifi/30-virtualmachine.yaml:119`, `40-networkpolicy.yaml:36` | `# legacy Network UI (still served)`, `# legacy UI` on port 8443 | load-bearing | **keep.** "legacy" here is the upstream product's own name for a port that is still served — it describes current behaviour, not history. Listed so a keyword-driven sweep does not delete it | no |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Per-directory counts (findings listed above)
|
||||||
|
|
||||||
|
| Directory | historical | load-bearing | ambiguous | total | sensitive-path? |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| `platform/` | 2 | 7 | 0 | 9 | no (except `kyverno/**`) |
|
||||||
|
| `infra/` | 3 | 5 | 0 | 8 | no (except `gitea/**`) |
|
||||||
|
| `catalog/` | 4 | 3 | 0 | 7 | no |
|
||||||
|
| `clusters/` | 2 | 5 | 0 | 7 | **yes** |
|
||||||
|
| `ci/` | 3 | 3 | 0 | 6 | **yes** |
|
||||||
|
| `conformance/` | 3 | 2 | 0 | 5 | **yes** |
|
||||||
|
| `.gitea/` | 2 | 3 | 0 | 5 | **yes** |
|
||||||
|
| `bootstrap/` | 1 | 1 | 1 | 3 | **yes** (×2) |
|
||||||
|
| `platform-config/` | 2 | 1 | 0 | 3 | **yes** |
|
||||||
|
| `apps/vms/` | 0 | 2 | 0 | 2 | no |
|
||||||
|
| **total** | **22** | **32** | **1** | **55** | 29 of 55 sensitive |
|
||||||
|
|
||||||
|
The `ambiguous` bucket is thin because I could rule on most rows against the reporter's
|
||||||
|
calibration. Two groups I am **deliberately not ruling on** and want the operator's call:
|
||||||
|
|
||||||
|
1. **`ci/tests/*.test.sh` headers** (5 files). These say things like "this suite exists
|
||||||
|
because `ci/gate.sh` used to set BLOCK=1 for a FAILED stage but…" and "a stage was added
|
||||||
|
to gate.sh by PR #17, never added here, so 15 of 50 assertions failed on `main`". For a
|
||||||
|
*regression test*, the regression it guards is arguably the test's subject rather than
|
||||||
|
fluff. Ruling these `historical` would strip the only record of what each assertion is
|
||||||
|
for. **Operator to decide.**
|
||||||
|
2. **In-repo `README.md` prose** (`platform/vpn-egress/README.md:91`,
|
||||||
|
`platform/crafty/README.md:52`, `infra/devcontainer/README.md:126`,
|
||||||
|
`infra/gitea/README.md:253`). These are documentation, not source comments, so arguably
|
||||||
|
already in the place the issue wants narrative to live. I did not audit them as findings.
|
||||||
|
|
||||||
|
### Suggested order for the follow-up PRs — cheapest and least sensitive first
|
||||||
|
|
||||||
|
1. **`catalog/` + re-render `apps/wave-*`** (7 findings, not sensitive). Highest value per
|
||||||
|
unit of review: it also fixes the generated copies under `apps/wave-*`. Must re-run
|
||||||
|
`python3 catalog/render.py --all catalog -o apps --wave-layout --no-image-automation`,
|
||||||
|
drop the not-onboarded outputs, and commit the goldens, or `render-golden` fails.
|
||||||
|
2. **`platform/` excluding `kyverno/**`** (9 findings, not sensitive). Biggest residual too
|
||||||
|
(107 candidates) — worth splitting by component (`monitoring/`, `longhorn/`, `crafty/`,
|
||||||
|
`headlamp/`, `vpn-egress/`) into 2–3 PRs rather than one sweep.
|
||||||
|
3. **`infra/` excluding `gitea/**`** (8 findings, not sensitive). `ansible/roles/*` is most
|
||||||
|
of it; `infra/plex/claim-ship.sh:29-32` (#10) is a clean one-paragraph deletion and a
|
||||||
|
good first commit.
|
||||||
|
4. **`apps/vms/`** (2 findings, not sensitive). Self-contained; #54 is the twin of #1, so do
|
||||||
|
it right after `platform/crafty/service-l4.yaml` while the reasoning is fresh.
|
||||||
|
5. **The dangling-pointer fix** (18 files, crosses sensitive and non-sensitive). Either
|
||||||
|
write one `decisions/` entry for the leaf-gating lesson and repoint all 18, or drop the
|
||||||
|
citations. Doing this *before* steps 6–9 means those PRs do not have to re-litigate it.
|
||||||
|
6. **`platform-config/kustomization.yaml`** (3 findings, **sensitive**). One file, three
|
||||||
|
edits — the smallest possible sensitive-path PR, so it is the cheapest review.
|
||||||
|
7. **`bootstrap/`** (3 findings, **sensitive**). Two files.
|
||||||
|
8. **`.gitea/`** (5 findings, **sensitive**). Keep #27 (`gate.yml:118-123`) in its own commit
|
||||||
|
— it documents a live, deliberate tolerance-widening and is the one most likely to be
|
||||||
|
misread as a behaviour change.
|
||||||
|
9. **`ci/`** then **`clusters/`** then **`conformance/`** (6 + 7 + 5, all **sensitive**,
|
||||||
|
largest residuals). Last, because each needs a non-author approving review or an operator
|
||||||
|
`/approve-override`, and because `ci/` edits touch the thing doing the approving.
|
||||||
|
|
||||||
|
A note on review cost: **29 of the 55 findings are on sensitive paths**, so roughly half
|
||||||
|
this work cannot merge on a green gate alone. Steps 1–4 (26 findings) are the portion that
|
||||||
|
can. That is the argument for the ordering above.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
Everything below was run in this container, offline, against the tree at
|
||||||
|
`466f63e` on branch `docs/source-comment-audit`.
|
||||||
|
|
||||||
|
- **The tree layout, against the brief's believed list.** `git ls-files` → **855** tracked
|
||||||
|
files. The brief's list was broadly right but differs in three ways worth recording:
|
||||||
|
(a) there is **no `scripts/` and no `tools/` directory**; (b) `docs/` **does exist in this
|
||||||
|
repo** (9 files: `docs/BULK-NFS-STORAGE.md` + `docs/plans/*.md`) and is *not* the sibling
|
||||||
|
spec repo's `docs/` — I audited neither, since both are prose, not source comments;
|
||||||
|
(c) `platform-config-cdi/` exists as a separate top-level directory (1 file, 0 candidate
|
||||||
|
hits).
|
||||||
|
- **Candidate volume, per directory**, with the keyword sweep quoted verbatim above:
|
||||||
|
`platform/` 107 hits / 56 files, `infra/` 75 / 44, `conformance/` 80 / 40 (**72 excluding
|
||||||
|
the 8 in `conformance/runs/`**), `clusters/` 43 / 19, `ci/` 42 / 15, `catalog/` 39 / 9,
|
||||||
|
`apps/` 18 (**15 under the generated `apps/wave-*`, 3 under hand-authored `apps/vms/`**),
|
||||||
|
`runbooks/` 11, `.gitea/` 10 / 5, `bootstrap/` 4 / 3, `platform-config/` 3 / 1,
|
||||||
|
`platform-config-cdi/` 0. Total ≈ **420 candidate hits in ≈190 files**.
|
||||||
|
- **`apps/wave-*` is generated and byte-enforced.** Read `ci/render-golden.sh`: the canonical
|
||||||
|
command is at `:21`/`:120`, and the comparison loop at `:136-150` iterates the *rendered*
|
||||||
|
output, requiring each golden to exist and to be byte-identical, `exit 1` on any diff.
|
||||||
|
Confirmed the duplication concretely: the "HARDENED (was a silent first-boot RACE)"
|
||||||
|
paragraph appears at both `catalog/gitea.yaml:100` and `apps/wave-0/gitea.yaml:130`.
|
||||||
|
Confirmed `apps/vms/` is *not* produced by that loop (the renderer emits `wave-N/` paths
|
||||||
|
only), so it is hand-authored and directly editable.
|
||||||
|
- **The dangling pointer.** `grep -ci "gating stall" DECISIONS-LOG.md` → **0**;
|
||||||
|
`grep -ci "Headlamp gating stall" DECISIONS-LOG.md` → **0**;
|
||||||
|
`grep -ril "gating stall" decisions/` → **no output**; `git grep -il "gating stall"` →
|
||||||
|
**18 files**. So 18 files cite a rationale entry that exists in neither the frozen archive
|
||||||
|
nor `decisions/`.
|
||||||
|
- **Every `decisions/...md` pointer I recommend exists.** Checked against `ls decisions/`
|
||||||
|
(71 entries + `README.md` + `ARCHIVE.sha256`): `2026-10-18-keycloak-passkey-reconciler.md`,
|
||||||
|
`2026-10-06-new-peer-ignores-template-comments.md`, `2026-10-01-quicksync-via-device-plugin.md`,
|
||||||
|
`2026-10-03-cilium-cgroup-hostroot.md`, `2026-10-05-oauth2-sidecar-scrape-verified.md`,
|
||||||
|
`2026-10-07-bulk-nfs-truenas.md`, `2026-10-30-renovate-auth.md`,
|
||||||
|
`2026-09-29-decisions-log-conflict-free.md`, `2026-10-04-pihole-multi-replica.md`,
|
||||||
|
`2026-10-04-pihole-declare-state.md` — all present. I recommend **no pointer target I did
|
||||||
|
not list in `ls` output**.
|
||||||
|
- **Archive-citation targets**, for the rows where I say a `DECISIONS-LOG` pointer does or
|
||||||
|
does not resolve: `"Remote Path"` 5 hits, `"Phase 3 VPN"` 10, `"Phase 4 image-automation"` 1,
|
||||||
|
`F-DNS1` 3, `F-OPS1` 1, `2026-09-01` 6 — all present in the 5,733-line archive.
|
||||||
|
`"Headlamp gating stall"` 0, as above.
|
||||||
|
- **The sensitive-path column is from the policy data, not the brief.** Read
|
||||||
|
`infra/gitea/approval-policy.json`: `sensitive_paths` = `ci/**`, `.gitea/**`,
|
||||||
|
`conformance/**`, `clusters/**`, `bootstrap/**`, `infra/gitea/**`, `platform/kyverno/**`,
|
||||||
|
`platform-config/kustomization.yaml`, `.sops.yaml`, `**/*.sops.yaml`, `**/rbac.yaml`,
|
||||||
|
`CLAUDE.md`. This matches the brief, and adds the root `.sops.yaml` and the explicit
|
||||||
|
`**/` glob forms. Note `**/*.sops.yaml` does **not** match `*.sops.yaml.example`, which is
|
||||||
|
why findings #50 and the `platform/headlamp/...sops.yaml.example` row are marked not
|
||||||
|
sensitive.
|
||||||
|
- **Every cited `path:line-range` was opened and read**, not matched and trusted. All 55
|
||||||
|
rows quote text I read with `sed -n` at the line numbers given, at `466f63e`. Line numbers
|
||||||
|
are from that commit and this PR changes no source file, so they remain valid on merge.
|
||||||
|
- **Secret-adjacent files**: no `*.sops.yaml` content is quoted anywhere in this report, and
|
||||||
|
no `.data`/`.stringData` value or key name appears. The two `.sops.yaml.example` rows
|
||||||
|
(#50, and `platform/headlamp/headlamp-oidc-credentials.sops.yaml.example:15` in the
|
||||||
|
unlisted set) are cited by path and line only. I ran no `sops -d`.
|
||||||
|
- **The gate.** Result recorded in the PR; see "What I did NOT verify" for the stages that
|
||||||
|
cannot run in this container.
|
||||||
|
- **`git diff --stat` against `main`**: one added file, `decisions/2026-10-31-source-comment-audit.md`,
|
||||||
|
and nothing else — which is the brief's "done means" condition.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **Anything about the live cluster. I hold no cluster credentials** — no kubeconfig and no
|
||||||
|
ServiceAccount token is mounted in this container, so `kubectl` cannot work here. This
|
||||||
|
audit is a pure text analysis of the tree and makes **no claim about the running
|
||||||
|
platform**. That is appropriate (no behaviour changes), but it also means every *factual*
|
||||||
|
assertion quoted inside the comments I audited — the measured Cilium lease holders, the
|
||||||
|
`longhorn_volume_robustness` series counts, the 192.168.2.30 NFS reachability, the k3s
|
||||||
|
version, the oauth2-proxy scrape targets — is **unverified by me**. I classified comments
|
||||||
|
by what they *talk about*, not by whether their measurements are still true. A rewrite PR
|
||||||
|
that restates one of those measurements as current fact should re-measure it first.
|
||||||
|
- **The two gate stages that cannot run in this container**, per the environment notes and
|
||||||
|
`CLAUDE.md` §3: `spec-immutability` (the sibling spec repo has no git remote and is not
|
||||||
|
reachable from the cluster) and `gitea-settings` (needs `SETTINGS_API_TOKEN`, which I
|
||||||
|
deliberately do not hold). So R-TEST-06 is **not** actually exercised on this run.
|
||||||
|
- **~365 of the ~420 candidate hits were not examined.** The per-directory table marks which
|
||||||
|
directories are full and which are sampled. For `platform/`, `infra/`, `conformance/`,
|
||||||
|
`clusters/`, `ci/` and `catalog/` the listed findings are **representative, not complete**,
|
||||||
|
and the `historical : load-bearing` ratio I report (22:32) is measured over the 55 rows I
|
||||||
|
judged, **not** over the whole corpus — do not project it onto the residual.
|
||||||
|
- **The keyword sweep is not a complete detector.** It finds English phrasings I thought of.
|
||||||
|
Historical narrative that avoids all of them — a dated war story with no "used to", a
|
||||||
|
stale rationale phrased entirely in the present tense, a comment that is simply wrong
|
||||||
|
about today — is **invisible to it**. The most dangerous category (a confidently
|
||||||
|
present-tense comment describing a superseded state) is precisely the one this method
|
||||||
|
cannot find. A complete audit needs a reader per directory, not a grep.
|
||||||
|
- **`runbooks/` (11 candidate hits), the repo-root markdown (`README.md`, `DESIGN-waves.md`),
|
||||||
|
and the in-repo `docs/` tree were not audited.** They are prose documentation rather than
|
||||||
|
source comments, so they fall outside the reporter's wording — but `CLAUDE.md` §1 already
|
||||||
|
records that `runbooks/` contains stale `git push origin main` instructions, so there is
|
||||||
|
known stale narrative there that this audit does not cover.
|
||||||
|
- **The sibling spec repo (`R-*` requirements, ADRs) was not read or assessed at all** — it
|
||||||
|
is not present in this container and is human-owned and immutable to agents (R-TEST-06).
|
||||||
|
Nothing here proposes a change to it. Where a comment cites a spec document
|
||||||
|
(e.g. `docs/06/R-STATE-01` in finding #44), I did **not** verify that the citation
|
||||||
|
resolves.
|
||||||
|
- **I did not verify that any recommended rewrite is correct, only that the current comment
|
||||||
|
is historical.** Each "rewrite" row names what I believe is the load-bearing residue; the
|
||||||
|
engineer doing the pruning must confirm that residue is still true before writing it as
|
||||||
|
present-tense fact. I wrote no replacement text.
|
||||||
|
- **Whether the operator agrees with my `historical` / `load-bearing` line.** I applied the
|
||||||
|
calibration from issue #97 literally plus the brief's one carve-out. The 22:32 split is a
|
||||||
|
judgement call, and the two groups in the `ambiguous` discussion above are explicitly
|
||||||
|
unresolved and need a human ruling before the `ci/tests/` and README rows are touched.
|
||||||
|
- **`conformance/runs/*/run.log` and `apps/wave-*` were excluded by my own judgement**, on
|
||||||
|
the structural grounds given (archived evidence; gate-enforced goldens). The operator has
|
||||||
|
not confirmed either exclusion.
|
||||||
|
- **Nothing here traces to an `R-*` requirement**, deliberately — this is housekeeping, per
|
||||||
|
the brief. I did not attempt to attribute one, and I did not check whether
|
||||||
|
`ci/touched-requirements.sh` expects an attribution for a `decisions/`-only change.
|
||||||
@@ -241,20 +241,25 @@ about runtime behaviour in this document is a design claim, not a measurement.
|
|||||||
`gitea.{{ k3s_base_domain }}` to the Gateway VIP in `node_baseline_hosts_pins`, the same
|
`gitea.{{ k3s_base_domain }}` to the Gateway VIP in `node_baseline_hosts_pins`, the same
|
||||||
pin the Keycloak issuer already carries, applied by
|
pin the Keycloak issuer already carries, applied by
|
||||||
`ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags hosts`;
|
`ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags hosts`;
|
||||||
(ii) the pull is still unauthenticated and no `imagePullSecret` exists in the tree — with
|
(ii) the pull must be authenticated — **now wired**: `pm-agent-registry`
|
||||||
the credential decision now being local Gitea accounts with PATs, that means a
|
(`dockerconfigjson`) is referenced by the Deployment and the committed worker Job
|
||||||
`dockerconfigjson` for an account whose token carries `read:package`. Only then does the
|
template, and the admission policy pins `imagePullSecrets` to exactly that name. The
|
||||||
digest matter. **Superseded 2026-10-29:** the build-and-push is no longer an operator step.
|
operator supplies the value, from a Gitea token that also carries `read:package`. Only
|
||||||
`.gitea/workflows/images.yaml` builds and verifies the image on rootless BuildKit
|
then does the digest matter. **Superseded 2026-10-29:** the build-and-push is no longer
|
||||||
(`platform/buildkit/`), publishes it on a push to `main`, and Renovate opens the one
|
an operator step. `.gitea/workflows/images.yaml` builds and verifies the image on rootless
|
||||||
PR that replaces both placeholder digests (`decisions/2026-10-29-image-pipeline.md`). The
|
BuildKit (`platform/buildkit/`) and publishes it on a push to `main`
|
||||||
same two blockers sit in front of `conformance-runner`, which shares the registry path and
|
(`decisions/2026-10-29-image-pipeline.md`); the first published digest is pinned in
|
||||||
has never been pulled.
|
`deployment.yaml` and `configmap.yaml`, and Renovate carries every later bump through the
|
||||||
|
`# renovate-digest:` markers (`decisions/2026-10-30-pm-agent-live.md`). The same
|
||||||
|
credential blocker sits in front of `conformance-runner`, which shares the registry path
|
||||||
|
and has never been pulled.
|
||||||
2. **Create the credentials** per `pm-agent-credentials.sops.yaml.example`, and **verify the
|
2. **Create the credentials** per `pm-agent-credentials.sops.yaml.example`, and **verify the
|
||||||
bot's permission is `write`, not `admin`** — that measurement is the control that keeps
|
bot's permission is `write`, not `admin`** — that measurement is the control that keeps
|
||||||
the agent out of its own approval gate.
|
the agent out of its own approval gate.
|
||||||
3. **First run with `PM_DRY_RUN=true` and `PM_REQUIRE_LABEL=agent`**, on one issue written
|
3. **First run with `PM_DRY_RUN=true` and `PM_REQUIRE_LABEL=agent`**, on one issue written
|
||||||
for the purpose. Read what the PM says before letting it write anything.
|
for the purpose. Read what the PM says before letting it write anything. *Done:* dry-run
|
||||||
|
triage of issue #97 read; `PM_DRY_RUN=false` on 2026-10-31
|
||||||
|
(`decisions/2026-10-31-pm-agent-go-live.md`), `PM_REQUIRE_LABEL=agent` kept.
|
||||||
4. **Tighten the admission image pin** from the `@sha256:*` wildcard to the literal digest.
|
4. **Tighten the admission image pin** from the `@sha256:*` wildcard to the literal digest.
|
||||||
One line, reviewed.
|
One line, reviewed.
|
||||||
5. **Narrow `world:443` to `toFQDNs: api.anthropic.com`** — after verifying Cilium's DNS
|
5. **Narrow `world:443` to `toFQDNs: api.anthropic.com`** — after verifying Cilium's DNS
|
||||||
|
|||||||
@@ -214,6 +214,12 @@ class Config:
|
|||||||
# ConfigMap without rebuilding the image.
|
# ConfigMap without rebuilding the image.
|
||||||
self.push_on_red = env("PM_PUSH_ON_RED_GATE", "false")
|
self.push_on_red = env("PM_PUSH_ON_RED_GATE", "false")
|
||||||
self.pm_timeout = env_int("PM_DECISION_TIMEOUT_SECONDS", 900)
|
self.pm_timeout = env_int("PM_DECISION_TIMEOUT_SECONDS", 900)
|
||||||
|
# The PM holds no tools, so the only thing a second turn can be spent on is the CLI's
|
||||||
|
# structured-output step re-asking the model after an answer that failed the schema.
|
||||||
|
# 1 is the long-standing value; it is a knob so that, if the failure envelope now
|
||||||
|
# logged by describe_cli_failure() says `error_max_turns`, the remedy is a ConfigMap
|
||||||
|
# change rather than another image build.
|
||||||
|
self.pm_max_turns = env_int("PM_DECISION_MAX_TURNS", 1)
|
||||||
# Opt-in gate. Empty (the default) means every open issue is in scope, which is the
|
# Opt-in gate. Empty (the default) means every open issue is in scope, which is the
|
||||||
# behaviour the service was asked for. Setting it to a label name — `agent` is the
|
# behaviour the service was asked for. Setting it to a label name — `agent` is the
|
||||||
# conventional one — narrows the agent to issues a human has explicitly handed it,
|
# conventional one — narrows the agent to issues a human has explicitly handed it,
|
||||||
@@ -231,6 +237,8 @@ class Config:
|
|||||||
|
|
||||||
if self.max_active_workers < 1:
|
if self.max_active_workers < 1:
|
||||||
die("PM_MAX_ACTIVE_WORKERS must be >= 1")
|
die("PM_MAX_ACTIVE_WORKERS must be >= 1")
|
||||||
|
if not 1 <= self.pm_max_turns <= 5:
|
||||||
|
die("PM_DECISION_MAX_TURNS must be between 1 and 5")
|
||||||
if "@sha256:" not in self.worker_image:
|
if "@sha256:" not in self.worker_image:
|
||||||
# The admission policy requires a digest too, so a tag here would produce a Job
|
# The admission policy requires a digest too, so a tag here would produce a Job
|
||||||
# that is created and then DENIED — a failure visible only in the dispatcher's
|
# that is created and then DENIED — a failure visible only in the dispatcher's
|
||||||
@@ -437,7 +445,7 @@ def ask_pm(cfg: Config, thread: str) -> dict:
|
|||||||
"--system-prompt-file", cfg.persona_file,
|
"--system-prompt-file", cfg.persona_file,
|
||||||
"--tools", "",
|
"--tools", "",
|
||||||
"--permission-prompts", "none",
|
"--permission-prompts", "none",
|
||||||
"--max-turns", "1",
|
"--max-turns", str(cfg.pm_max_turns),
|
||||||
"--output-format", "json",
|
"--output-format", "json",
|
||||||
"--json-schema", json.dumps(DECISION_SCHEMA),
|
"--json-schema", json.dumps(DECISION_SCHEMA),
|
||||||
"--no-session-persistence",
|
"--no-session-persistence",
|
||||||
@@ -445,10 +453,76 @@ def ask_pm(cfg: Config, thread: str) -> dict:
|
|||||||
proc = subprocess.run(cmd, input=thread, capture_output=True, text=True,
|
proc = subprocess.run(cmd, input=thread, capture_output=True, text=True,
|
||||||
timeout=cfg.pm_timeout)
|
timeout=cfg.pm_timeout)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise RuntimeError(f"claude exited {proc.returncode}: {proc.stderr.strip()[:800]}")
|
raise RuntimeError(describe_cli_failure(proc.returncode, proc.stdout, proc.stderr))
|
||||||
return extract_decision(proc.stdout)
|
return extract_decision(proc.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
# Anything that could be a credential in the CLI's output. The CLI does not normally echo its
|
||||||
|
# token, but an API error body or a proxy's HTML page is not something to bet a leak on. Both
|
||||||
|
# schemes are matched (the http one is the lesson of a scheme-specific redaction that leaked
|
||||||
|
# ~/.git-credentials), and `token ...` is Gitea's own header scheme.
|
||||||
|
_SECRET_PATTERNS = [
|
||||||
|
(re.compile(r"sk-ant-[A-Za-z0-9_\-]+"), "sk-ant-[REDACTED]"),
|
||||||
|
(re.compile(r"(?i)\b(bearer|token)(\s+|=|:\s*)[A-Za-z0-9._~+/=\-]{8,}"), r"\1\2[REDACTED]"),
|
||||||
|
(re.compile(r"(?i)(https?://)[^/\s:@]+:[^/\s@]+@"), r"\1[REDACTED]@"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def redact(text: str) -> str:
|
||||||
|
for pat, repl in _SECRET_PATTERNS:
|
||||||
|
text = pat.sub(repl, text)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def describe_cli_failure(rc: int, stdout: str, stderr: str, limit: int = 800) -> str:
|
||||||
|
"""Say WHY `claude --print` failed, in the dispatcher log, redacted.
|
||||||
|
|
||||||
|
WHY THIS EXISTS: with `--output-format json` the CLI reports its own failures — an API
|
||||||
|
error, a usage limit, `error_max_turns`, a schema the model could not satisfy — as the
|
||||||
|
JSON result envelope on STDOUT (`is_error`, `subtype`, `result`), and writes nothing to
|
||||||
|
stderr. Logging only stderr therefore produced `claude exited 1: ` with an empty message
|
||||||
|
on 2026-10-31, twice, with the cause sitting unread in stdout. So: the envelope's
|
||||||
|
diagnostic fields first, then the raw tails of both streams.
|
||||||
|
"""
|
||||||
|
parts = [f"claude exited {rc}"]
|
||||||
|
out = (stdout or "").strip()
|
||||||
|
env = None
|
||||||
|
if out:
|
||||||
|
try:
|
||||||
|
env = json.loads(out)
|
||||||
|
except ValueError:
|
||||||
|
# A stream of JSON lines, or a log line ahead of the envelope: the last line that
|
||||||
|
# parses as an object is the result.
|
||||||
|
for line in reversed(out.splitlines()):
|
||||||
|
try:
|
||||||
|
cand = json.loads(line)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if isinstance(cand, dict):
|
||||||
|
env = cand
|
||||||
|
break
|
||||||
|
if isinstance(env, dict):
|
||||||
|
fields = []
|
||||||
|
for key in ("subtype", "is_error", "api_error_status", "num_turns", "duration_ms",
|
||||||
|
"duration_api_ms", "total_cost_usd", "stop_reason"):
|
||||||
|
if key in env:
|
||||||
|
fields.append(f"{key}={env[key]!r}")
|
||||||
|
if fields:
|
||||||
|
parts.append("envelope: " + " ".join(fields))
|
||||||
|
for key in ("result", "error", "errors", "message"):
|
||||||
|
if env.get(key):
|
||||||
|
val = env[key] if isinstance(env[key], str) else json.dumps(env[key])
|
||||||
|
parts.append(f"{key}: {val[:limit]}")
|
||||||
|
if out and not isinstance(env, dict):
|
||||||
|
parts.append(f"stdout tail: {out[-limit:]}")
|
||||||
|
err_text = (stderr or "").strip()
|
||||||
|
if err_text:
|
||||||
|
parts.append(f"stderr tail: {err_text[-limit:]}")
|
||||||
|
if len(parts) == 1:
|
||||||
|
parts.append("(no output on stdout or stderr)")
|
||||||
|
return redact(" | ".join(parts))
|
||||||
|
|
||||||
|
|
||||||
def extract_decision(stdout: str) -> dict:
|
def extract_decision(stdout: str) -> dict:
|
||||||
"""Pull the decision object out of whatever `claude --print` wrapped it in.
|
"""Pull the decision object out of whatever `claude --print` wrapped it in.
|
||||||
|
|
||||||
@@ -501,7 +575,7 @@ def extract_decision(stdout: str) -> dict:
|
|||||||
obj = as_obj(c)
|
obj = as_obj(c)
|
||||||
if obj and isinstance(obj.get("action"), str):
|
if obj and isinstance(obj.get("action"), str):
|
||||||
return obj
|
return obj
|
||||||
raise RuntimeError(f"no decision object in claude output: {stdout[:400]!r}")
|
raise RuntimeError(redact(f"no decision object in claude output: {stdout[-800:]!r}"))
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- thread
|
# --------------------------------------------------------------------------- thread
|
||||||
@@ -922,6 +996,8 @@ def selftest_template(path: str) -> None:
|
|||||||
assert pod.get("automountServiceAccountToken") is False, \
|
assert pod.get("automountServiceAccountToken") is False, \
|
||||||
"template must not mount a ServiceAccount token"
|
"template must not mount a ServiceAccount token"
|
||||||
assert pod.get("restartPolicy") == "Never", "a retried agent would double-push"
|
assert pod.get("restartPolicy") == "Never", "a retried agent would double-push"
|
||||||
|
assert [x.get("name") for x in pod.get("imagePullSecrets") or []] == ["pm-agent-registry"], \
|
||||||
|
"the template must name exactly the committed pull secret (admission pins it)"
|
||||||
assert len(pod.get("containers") or []) == 1, "exactly one container"
|
assert len(pod.get("containers") or []) == 1, "exactly one container"
|
||||||
c = pod["containers"][0]
|
c = pod["containers"][0]
|
||||||
assert c.get("command") == ["/usr/local/bin/pm-worker"], "command must be the pinned entrypoint"
|
assert c.get("command") == ["/usr/local/bin/pm-worker"], "command must be the pinned entrypoint"
|
||||||
@@ -986,8 +1062,25 @@ def selftest() -> int:
|
|||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
check(True, f"bad output refused: {label}")
|
check(True, f"bad output refused: {label}")
|
||||||
|
|
||||||
|
# -- a failed CLI run must say why, and must not leak ---------------------
|
||||||
|
# The 2026-10-31 failure: rc 1, empty stderr, the cause in the stdout envelope.
|
||||||
|
env_err = json.dumps({"type": "result", "subtype": "error_max_turns", "is_error": True,
|
||||||
|
"num_turns": 2, "result": "API Error: 429 rate_limit_error"})
|
||||||
|
msg = describe_cli_failure(1, env_err, "")
|
||||||
|
check("error_max_turns" in msg and "429" in msg,
|
||||||
|
"CLI failure: stdout envelope surfaced when stderr is empty")
|
||||||
|
msg = describe_cli_failure(1, "", "boom on stderr")
|
||||||
|
check("boom on stderr" in msg, "CLI failure: stderr tail surfaced")
|
||||||
|
msg = describe_cli_failure(1, "", "")
|
||||||
|
check("no output" in msg, "CLI failure: silence is named, not an empty message")
|
||||||
|
msg = describe_cli_failure(1, "plain text sk-ant-oat01-abcdefABCDEF_123 here",
|
||||||
|
"Authorization: Bearer abcdefghijklmnop "
|
||||||
|
"http://u:[email protected]/x")
|
||||||
|
check("sk-ant-oat01" not in msg and "abcdefghijklmnop" not in msg and ":pw@" not in msg,
|
||||||
|
"CLI failure: token, bearer and URL credential redacted")
|
||||||
|
|
||||||
# -- the committed Job template -----------------------------------------
|
# -- the committed Job template -----------------------------------------
|
||||||
tmpl = os.environ.get("PM_WORKER_JOB_TEMPLATE", "/etc/pm-agent/worker-job-template.yaml")
|
tmpl =os.environ.get("PM_WORKER_JOB_TEMPLATE", "/etc/pm-agent/worker-job-template.yaml")
|
||||||
try:
|
try:
|
||||||
selftest_template(tmpl)
|
selftest_template(tmpl)
|
||||||
check(True, f"worker Job template satisfies its contract ({tmpl})")
|
check(True, f"worker Job template satisfies its contract ({tmpl})")
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
# digest. For a LOCAL build on a local toolbox, override it:
|
# digest. For a LOCAL build on a local toolbox, override it:
|
||||||
# infra/devcontainer/dev.sh build && AGENT_BASE_IMAGE=home-k3s-dev:local infra/agent/build-image.sh
|
# infra/devcontainer/dev.sh build && AGENT_BASE_IMAGE=home-k3s-dev:local infra/agent/build-image.sh
|
||||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/toolbox tag=main
|
||||||
AGENT_BASE_IMAGE=gitea.home.keithjackson.dev/home-network/toolbox@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
AGENT_BASE_IMAGE=gitea.home.keithjackson.dev/home-network/toolbox@sha256:d435fabd6fc43bf5bb4de0c1ea8c4b19d8cea5c858679bd1675ccbed471b528a
|
||||||
|
|
||||||
# Node.js — the runtime for the Claude Code CLI, installed from the Fedora base's module
|
# Node.js — the runtime for the Claude Code CLI, installed from the Fedora base's module
|
||||||
# stream. A major version, not a patch: the CLI supports the active LTS line and the
|
# stream. A major version, not a patch: the CLI supports the active LTS line and the
|
||||||
|
|||||||
@@ -68,6 +68,13 @@ spec:
|
|||||||
# guarantee it cannot use one is for it not to have one.
|
# guarantee it cannot use one is for it not to have one.
|
||||||
automountServiceAccountToken: false
|
automountServiceAccountToken: false
|
||||||
serviceAccountName: pm-agent-worker
|
serviceAccountName: pm-agent-worker
|
||||||
|
# The kubelet's pull credential, the same one the dispatcher uses. Pinned by the
|
||||||
|
# admission policy to exactly this name: `imagePullSecrets` is a pod-spec field, so a
|
||||||
|
# Job could otherwise name any Secret in the namespace and have the kubelet read it
|
||||||
|
# before a container ever starts. The image is digest-pinned, so the leverage is small
|
||||||
|
# — but "only the committed shape" is the rule this template exists to express.
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
securityContext:
|
securityContext:
|
||||||
runAsNonRoot: true
|
runAsNonRoot: true
|
||||||
runAsUser: 65532
|
runAsUser: 65532
|
||||||
|
|||||||
@@ -19,7 +19,9 @@ Docker Hub images were purged (DECISIONS-LOG **F-KY1**) → that path ImagePullB
|
|||||||
3-replica **`longhorn`** StorageClass (DR-critical, R-IDP-05), with declarative backup hooks.
|
3-replica **`longhorn`** StorageClass (DR-critical, R-IDP-05), with declarative backup hooks.
|
||||||
|
|
||||||
quay.io + ghcr.io are unaffected by the Bitnami purge. Operator manifests are pinned upstream release
|
quay.io + ghcr.io are unaffected by the Bitnami purge. Operator manifests are pinned upstream release
|
||||||
files pulled as remote kustomize bases (the kubevirt-stub pattern); Renovate maintains the version.
|
files pulled as remote kustomize bases (the kubevirt-stub pattern). Renovate tracks the version via
|
||||||
|
`# renovate:` markers on those URLs (since 2026-10-30 — before that it could not see them) and never
|
||||||
|
auto-merges it; see [Upgrading Keycloak](#upgrading-keycloak-the-operator-is-the-version).
|
||||||
|
|
||||||
## Layout (controllers / configs split — F-MON1)
|
## Layout (controllers / configs split — F-MON1)
|
||||||
|
|
||||||
@@ -360,6 +362,38 @@ it on the BOUND flow, after go-live, in order:
|
|||||||
A failed passkey ceremony is usually silent in the server log — the browser refuses before Keycloak
|
A failed passkey ceremony is usually silent in the server log — the browser refuses before Keycloak
|
||||||
is involved — so step 2 failing with no log entry is the expected signature of a wrong RP ID.
|
is involved — so step 2 failing with no log entry is the expected signature of a wrong RP ID.
|
||||||
|
|
||||||
|
## Upgrading Keycloak: the operator IS the version
|
||||||
|
|
||||||
|
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
||||||
|
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
||||||
|
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
||||||
|
released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
|
||||||
|
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
||||||
|
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
||||||
|
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
||||||
|
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
||||||
|
|
||||||
|
**An operator bump requires re-checking the passkey reconciler's assumptions.**
|
||||||
|
`passkey-reconciler.yaml` (and `ci/render-keycloak-passkey-policy.py`, which generates its desired
|
||||||
|
state) is written against **Keycloak 26.0.6 Admin API behaviour read from that tag's source** —
|
||||||
|
grep it for `26.0.6`. The gate cannot catch a break: it is static and never runs the reconciler
|
||||||
|
against a server. Before merging a bump, diff the new tag against 26.0.6 for at least:
|
||||||
|
|
||||||
|
- `AuthenticationManagementResource.recurseExecutions` — does a sub-flow row still carry its name
|
||||||
|
in `displayName` (not `alias`)? The reconciler's flow matching depends on it.
|
||||||
|
- `updateExecutions` / execution-requirement updates — same request shape and id resolution?
|
||||||
|
- the WebAuthn Passwordless policy — is it still gated all-or-nothing on a non-empty
|
||||||
|
`webAuthnPolicyPasswordlessRpEntityName`, and does a realm `PUT` still REPLACE rather than merge
|
||||||
|
`requiredActions`? (The strict read-modify-write exists because of both.)
|
||||||
|
- authenticator-config alias handling for copied flows.
|
||||||
|
|
||||||
|
Then, after the merge reconciles (operator actions): watch the `Keycloak` CR reach Ready and both
|
||||||
|
pods roll; run the reconciler once **with `--audit`** (see "Passkeys" above for the one-off Job)
|
||||||
|
and confirm it reports convergence with no write; confirm the client-secret and client-URL
|
||||||
|
reconcilers' next runs succeed; log in to one SSO app and run one `kubectl --context k3s-prod`
|
||||||
|
OIDC command. Roll back by reverting the PR — but a server minor may migrate the database
|
||||||
|
schema on start, so take/confirm a fresh CNPG backup before merging.
|
||||||
|
|
||||||
## Exceptions that keep native auth (do not force SSO) — ADR-0014
|
## Exceptions that keep native auth (do not force SSO) — ADR-0014
|
||||||
|
|
||||||
Plex/Seerr (Plex auth), Minecraft (Mojang/Microsoft), UniFi (native).
|
Plex/Seerr (Plex auth), Minecraft (Mojang/Microsoft), UniFi (native).
|
||||||
@@ -21,8 +21,17 @@ namespace: keycloak
|
|||||||
resources:
|
resources:
|
||||||
- namespace.yaml
|
- namespace.yaml
|
||||||
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
||||||
|
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
||||||
|
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
||||||
|
# is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||||
|
# marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
|
||||||
|
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
||||||
|
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
||||||
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||||
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||||
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml
|
||||||
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
||||||
- keycloak-admin-credentials.sops.yaml
|
- keycloak-admin-credentials.sops.yaml
|
||||||
|
|||||||
@@ -111,6 +111,12 @@ spec:
|
|||||||
=(hostNetwork): "false"
|
=(hostNetwork): "false"
|
||||||
=(hostPID): "false"
|
=(hostPID): "false"
|
||||||
=(hostIPC): "false"
|
=(hostIPC): "false"
|
||||||
|
# The kubelet reads this Secret BEFORE any container starts, so it is part
|
||||||
|
# of the shape even though no container references it. A pattern on an array
|
||||||
|
# asserts "at least one element matches", which is enough to REQUIRE the
|
||||||
|
# committed entry; rule 7 is what forbids a second one.
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
containers:
|
containers:
|
||||||
- image: "gitea.home.keithjackson.dev/home-network/agent-runner@sha256:*"
|
- image: "gitea.home.keithjackson.dev/home-network/agent-runner@sha256:*"
|
||||||
command:
|
command:
|
||||||
@@ -336,6 +342,54 @@ spec:
|
|||||||
# dispatcher itself uses envFrom (it is a Deployment, not a Job, and not matched by this
|
# dispatcher itself uses envFrom (it is a Deployment, not a Job, and not matched by this
|
||||||
# policy); a worker must not.
|
# policy); a worker must not.
|
||||||
# =====================================================================================
|
# =====================================================================================
|
||||||
|
# =====================================================================================
|
||||||
|
# Rule 7 — exactly one imagePullSecret, and it is the committed one.
|
||||||
|
#
|
||||||
|
# `imagePullSecrets` is a pod-spec field that no container references, which is precisely
|
||||||
|
# why it is easy to leave unread — the same shape of omission as `initContainers` and
|
||||||
|
# `env` before it. The kubelet reads the named Secret, in this namespace, before any
|
||||||
|
# container starts. A Job naming a different one would have the kubelet read a Secret the
|
||||||
|
# worker was never meant to touch.
|
||||||
|
#
|
||||||
|
# The leverage is genuinely small: the image is digest-pinned, so a different credential
|
||||||
|
# cannot change WHAT runs, and the only Secrets in this namespace are the agent's own.
|
||||||
|
# It is pinned anyway, because "only the committed shape" is the whole premise of this
|
||||||
|
# policy, and because the cost of the rule is four lines while the cost of rediscovering
|
||||||
|
# the omission is an incident.
|
||||||
|
#
|
||||||
|
# Rule 1's pattern REQUIRES the committed entry; this forbids any OTHER entry and any
|
||||||
|
# second entry. A foreach over an absent list yields no result, which is why both halves
|
||||||
|
# are needed.
|
||||||
|
# =====================================================================================
|
||||||
|
- name: worker-pull-secret-must-be-the-committed-one
|
||||||
|
match:
|
||||||
|
all:
|
||||||
|
- resources:
|
||||||
|
kinds:
|
||||||
|
- Job
|
||||||
|
namespaces:
|
||||||
|
- pm-agent
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: pm-agent
|
||||||
|
namespace: pm-agent
|
||||||
|
validate:
|
||||||
|
message: >-
|
||||||
|
A worker Job may declare exactly ONE imagePullSecret, named `pm-agent-registry`.
|
||||||
|
The kubelet reads that Secret before any container starts, so naming another one
|
||||||
|
reaches a credential the worker was never meant to touch.
|
||||||
|
foreach:
|
||||||
|
- list: "request.object.spec.template.spec.imagePullSecrets"
|
||||||
|
deny:
|
||||||
|
conditions:
|
||||||
|
any:
|
||||||
|
- key: "{{ length(request.object.spec.template.spec.imagePullSecrets) }}"
|
||||||
|
operator: NotEquals
|
||||||
|
value: 1
|
||||||
|
- key: "{{ element.name }}"
|
||||||
|
operator: AnyNotIn
|
||||||
|
value: ["pm-agent-registry"]
|
||||||
|
|
||||||
- name: worker-may-not-use-envfrom
|
- name: worker-may-not-use-envfrom
|
||||||
match:
|
match:
|
||||||
all:
|
all:
|
||||||
|
|||||||
@@ -56,6 +56,11 @@ results:
|
|||||||
kind: Job
|
kind: Job
|
||||||
resources: [good-run, real-template]
|
resources: [good-run, real-template]
|
||||||
result: pass
|
result: pass
|
||||||
|
- policy: pm-agent-job-shape
|
||||||
|
rule: worker-pull-secret-must-be-the-committed-one
|
||||||
|
kind: Job
|
||||||
|
resources: [good-run, real-template]
|
||||||
|
result: pass
|
||||||
|
|
||||||
# ---------------- THE escalation, and the shape rule's other refusals ----------------
|
# ---------------- THE escalation, and the shape rule's other refusals ----------------
|
||||||
# `evil-command` is the case the whole model exists for: right namespace, right creator,
|
# `evil-command` is the case the whole model exists for: right namespace, right creator,
|
||||||
@@ -73,6 +78,14 @@ results:
|
|||||||
- host-namespaces
|
- host-namespaces
|
||||||
- restart-onfailure
|
- restart-onfailure
|
||||||
- second-container
|
- second-container
|
||||||
|
- no-pull-secret
|
||||||
|
result: fail
|
||||||
|
|
||||||
|
# ---------------- the pull secret: a second one, or a different one ----------------
|
||||||
|
- policy: pm-agent-job-shape
|
||||||
|
rule: worker-pull-secret-must-be-the-committed-one
|
||||||
|
kind: Job
|
||||||
|
resources: [extra-pull-secret, wrong-pull-secret]
|
||||||
result: fail
|
result: fail
|
||||||
|
|
||||||
# ---------------- args, and the second container ----------------
|
# ---------------- args, and the second container ----------------
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ spec:
|
|||||||
restartPolicy: Never
|
restartPolicy: Never
|
||||||
automountServiceAccountToken: false
|
automountServiceAccountToken: false
|
||||||
serviceAccountName: pm-agent-worker
|
serviceAccountName: pm-agent-worker
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
securityContext:
|
securityContext:
|
||||||
runAsNonRoot: true
|
runAsNonRoot: true
|
||||||
runAsUser: 65532
|
runAsUser: 65532
|
||||||
@@ -95,6 +97,8 @@ spec:
|
|||||||
restartPolicy: Never
|
restartPolicy: Never
|
||||||
automountServiceAccountToken: false
|
automountServiceAccountToken: false
|
||||||
serviceAccountName: pm-agent-worker
|
serviceAccountName: pm-agent-worker
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
containers:
|
containers:
|
||||||
- name: worker
|
- name: worker
|
||||||
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
|
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
|
||||||
@@ -492,3 +496,75 @@ spec:
|
|||||||
readOnlyRootFilesystem: true
|
readOnlyRootFilesystem: true
|
||||||
envFrom:
|
envFrom:
|
||||||
- secretRef: { name: pm-agent-credentials }
|
- secretRef: { name: pm-agent-credentials }
|
||||||
|
---
|
||||||
|
# A SECOND pull secret beside the committed one. containers[0] is perfectly compliant, and
|
||||||
|
# rule 1's array pattern is satisfied by the first element — which is why the count is
|
||||||
|
# asserted separately.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: extra-pull-secret
|
||||||
|
namespace: pm-agent
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
serviceAccountName: pm-agent-worker
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
|
- name: pm-agent-credentials
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
|
||||||
|
command: ["/usr/local/bin/pm-worker"]
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
---
|
||||||
|
# A DIFFERENT pull secret — the kubelet would read a Secret the worker was never meant to
|
||||||
|
# touch, before any container starts.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: wrong-pull-secret
|
||||||
|
namespace: pm-agent
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
serviceAccountName: pm-agent-worker
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-credentials
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
|
||||||
|
command: ["/usr/local/bin/pm-worker"]
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
---
|
||||||
|
# NO pull secret at all. Rule 7's foreach yields nothing over an absent list, so this is
|
||||||
|
# rule 1's pattern to refuse — which is why the requirement lives in both.
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: no-pull-secret
|
||||||
|
namespace: pm-agent
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
serviceAccountName: pm-agent-worker
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:aaaabbbbccccddddeeeeffff0000111122223333444455556666777788889999
|
||||||
|
command: ["/usr/local/bin/pm-worker"]
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
+47
-21
@@ -46,15 +46,18 @@ waive the sensitive-path human review on its own pull request.
|
|||||||
Both are deliberate, and both fail **visibly** rather than silently:
|
Both are deliberate, and both fail **visibly** rather than silently:
|
||||||
|
|
||||||
1. **The image is published by CI, not by a person.** `platform/pm-agent/deployment.yaml` and
|
1. **The image is published by CI, not by a person.** `platform/pm-agent/deployment.yaml` and
|
||||||
`PM_WORKER_IMAGE` in `configmap.yaml` carry an all-zero digest placeholder until the first
|
`PM_WORKER_IMAGE` in `configmap.yaml` first carried the run-340 publish
|
||||||
publish. `.gitea/workflows/images.yaml` builds `agent-runner` on rootless BuildKit
|
(`sha-523cd9118f95`), whose baked template predated the `imagePullSecrets` entry admission
|
||||||
|
requires, so the service ran with `PM_DRY_RUN=true`. PR #96 pinned the run-383 digest
|
||||||
|
(`df36d9b`, `sha-ddd04d460ef6`), whose template carries the entry, and dry run was turned
|
||||||
|
off on 2026-10-31 (`decisions/2026-10-31-pm-agent-go-live.md`); every later bump is
|
||||||
|
Renovate's. `.gitea/workflows/images.yaml` builds `agent-runner` on rootless BuildKit
|
||||||
(`platform/buildkit/`) — the same Containerfile, and the same `verify` stage, that
|
(`platform/buildkit/`) — the same Containerfile, and the same `verify` stage, that
|
||||||
`infra/agent/build-image.sh` builds locally — and on a push to `main` publishes it to
|
`infra/agent/build-image.sh` builds locally — and on a push to `main` publishes it to
|
||||||
`gitea.home.keithjackson.dev/home-network/agent-runner` under `sha-<commit>` and `main`.
|
`gitea.home.keithjackson.dev/home-network/agent-runner` under `sha-<commit>` and `main`.
|
||||||
Renovate then follows the `main` tag through the `# renovate-digest:` marker above **both**
|
Renovate then follows the `main` tag through the `# renovate-digest:` marker above **both**
|
||||||
lines and moves them together in one pull request. Nobody builds-and-pushes this image from
|
lines and moves them together in one pull request. Nobody builds-and-pushes this image from
|
||||||
a workstation, and nobody pastes a digest. Until the first publish, and until that PR
|
a workstation. The one operator prerequisite on the publishing side is the
|
||||||
merges: `ImagePullBackOff`. The one operator prerequisite on the publishing side is the
|
|
||||||
`REGISTRY_PUSH_TOKEN` Actions secret (see the header of `.gitea/workflows/images.yaml`);
|
`REGISTRY_PUSH_TOKEN` Actions secret (see the header of `.gitea/workflows/images.yaml`);
|
||||||
without it the publish job fails loudly rather than skipping.
|
without it the publish job fails loudly rather than skipping.
|
||||||
|
|
||||||
@@ -78,21 +81,36 @@ Both are deliberate, and both fail **visibly** rather than silently:
|
|||||||
inventory, never typed on a node. Apply it scoped to that one task:
|
inventory, never typed on a node. Apply it scoped to that one task:
|
||||||
`cd infra/ansible && ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod
|
`cd infra/ansible && ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod
|
||||||
--tags hosts`.
|
--tags hosts`.
|
||||||
- **The pull is still unauthenticated** once the node reaches the right endpoint. An
|
- **The pull must be authenticated** once the node reaches the right endpoint: an
|
||||||
unauthenticated client gets nothing from this Gitea's registry, and there is no
|
unauthenticated client gets nothing from this registry. **Now wired** —
|
||||||
`imagePullSecret` anywhere in the tree. Consistent with the agent credential decision
|
`pm-agent-registry` is a `dockerconfigjson` Secret referenced by both the Deployment and
|
||||||
(local Gitea accounts with PATs), the fix is a `dockerconfigjson` for a local Gitea
|
the committed worker Job template, and the admission policy pins it to exactly that
|
||||||
account whose token carries `read:package`, referenced from the Deployment and the
|
name. What is left to you is the value: the Gitea token must carry **`read:package`** in
|
||||||
committed worker Job template — not built yet, because minting that token is yours.
|
addition to `read:user`, `write:repository` and `write:issue`, and the dockerconfigjson is built from
|
||||||
|
it. Steps:
|
||||||
|
[`pm-agent-registry.sops.yaml.example`](./pm-agent-registry.sops.yaml.example).
|
||||||
|
|
||||||
The same two apply to `conformance-runner`: same registry path, same publish-then-Renovate
|
The same two apply to `conformance-runner`: same registry path, same publish-then-Renovate
|
||||||
model, never pulled either.
|
model, never pulled either.
|
||||||
2. **The credentials are placeholders.** `pm-agent-credentials.sops.yaml` is encrypted to the
|
2. **The credentials are placeholders — two Secrets, one token.** Both are encrypted to the
|
||||||
repository's age recipient but holds `REPLACE-ME-…` strings. The hand-off —
|
repository's age recipient and both hold `REPLACE-ME-…` strings:
|
||||||
which Gitea account to create, which scopes, and why it must *not* be an administrator —
|
|
||||||
is in [`pm-agent-credentials.sops.yaml.example`](./pm-agent-credentials.sops.yaml.example).
|
| Secret | type | read by | hand-off |
|
||||||
Until then the dispatcher CrashLoops with `GET /user returned no login — is GITEA_TOKEN
|
|---|---|---|---|
|
||||||
valid?`.
|
| `pm-agent-credentials` | Opaque | the application, as env | [`.example`](./pm-agent-credentials.sops.yaml.example) |
|
||||||
|
| `pm-agent-registry` | dockerconfigjson | the **kubelet**, before any container starts | [`.example`](./pm-agent-registry.sops.yaml.example) |
|
||||||
|
|
||||||
|
They hold the **same** Gitea token in two formats, because a Secret cannot be both — the
|
||||||
|
same one-value-in-two-files pattern the Keycloak client secret already uses here. So mint
|
||||||
|
it once, with all four scopes: `read:package` (the pull), `read:user` (the dispatcher's
|
||||||
|
startup `GET /user` — Gitea 1.24.3 scopes the whole `/user` group), `write:repository`
|
||||||
|
(clone, branch push, pull request) and `write:issue` (issue comments). The account also
|
||||||
|
needs package *permission*, not just the scope: a `pm-agent-pull` org team with Packages
|
||||||
|
= Read and no repositories (steps in the registry `.example`). The first `.example`
|
||||||
|
explains which account to create and why it must *not* be an administrator.
|
||||||
|
|
||||||
|
Until they are filled in: the pull fails `401`, and if you fix only the pull, the
|
||||||
|
dispatcher CrashLoops with `GET /user returned no login — is GITEA_TOKEN valid?`.
|
||||||
|
|
||||||
Its Flux Kustomization is `wait: false` precisely because of those two states: a workload
|
Its Flux Kustomization is `wait: false` precisely because of those two states: a workload
|
||||||
that is *expected* to be unhealthy for a while must never be able to gate anything.
|
that is *expected* to be unhealthy for a while must never be able to gate anything.
|
||||||
@@ -155,13 +173,14 @@ Job from a committed template. **The model's output is data, never a command.**
|
|||||||
| **cannot** | clone from or push to anywhere but `gitea.gitea.svc.cluster.local:3000`: `pm-worker` validates `PM_GITEA_URL` against a one-entry allowlist baked into the image. In-cluster consumers reach a service by its cluster-local name — a different path by design from how a client outside the cluster reaches it — so the external hostname is not an accepted value |
|
| **cannot** | clone from or push to anywhere but `gitea.gitea.svc.cluster.local:3000`: `pm-worker` validates `PM_GITEA_URL` against a one-entry allowlist baked into the image. In-cluster consumers reach a service by its cluster-local name — a different path by design from how a client outside the cluster reaches it — so the external hostname is not an accepted value |
|
||||||
| **cannot** | authenticate a `git push` **during the model session**: `GIT_ASKPASS` is unset for its duration, so the restriction is structural rather than a request in a prompt |
|
| **cannot** | authenticate a `git push` **during the model session**: `GIT_ASKPASS` is unset for its duration, so the restriction is structural rather than a request in a prompt |
|
||||||
| **cannot** | be created in any shape but the committed one — see the admission policy |
|
| **cannot** | be created in any shape but the committed one — see the admission policy |
|
||||||
|
| **cannot** | have the kubelet read any Secret but `pm-agent-registry` on its behalf: `imagePullSecrets` is pinned to exactly that one name |
|
||||||
|
|
||||||
### What is *not* defended, stated plainly
|
### What is *not* defended, stated plainly
|
||||||
|
|
||||||
- **The worker's own credentials are inside the model's reach.** The session runs as the same
|
- **The worker's own credentials are inside the model's reach.** The session runs as the same
|
||||||
uid in the same container as the wrapper, so it can read the environment, including its
|
uid in the same container as the wrapper, so it can read the environment, including its
|
||||||
`GITEA_TOKEN` and `CLAUDE_CODE_OAUTH_TOKEN`. There is no isolation between them and none is
|
`GITEA_TOKEN` and `CLAUDE_CODE_OAUTH_TOKEN`. There is no isolation between them and none is
|
||||||
claimed. What bounds it: the Gitea token is scoped `write:repository, write:issue` on **one
|
claimed. What bounds it: the Gitea token is scoped `read:user, write:repository, write:issue, read:package` on **one
|
||||||
repository** with no admin permission, and egress reaches only this Gitea and tcp/443.
|
repository** with no admin permission, and egress reaches only this Gitea and tcp/443.
|
||||||
- **`world:443` is wider than `api.anthropic.com`.** Cilium can express the tighter rule
|
- **`world:443` is wider than `api.anthropic.com`.** Cilium can express the tighter rule
|
||||||
(`toFQDNs`) but nothing in this repository uses FQDN policy and its prerequisite (Cilium's
|
(`toFQDNs`) but nothing in this repository uses FQDN policy and its prerequisite (Cilium's
|
||||||
@@ -200,14 +219,15 @@ Two things answer it, and neither is sufficient alone:
|
|||||||
`command` with no args, that powerless SA, no mounted token, `restartPolicy: Never`,
|
`command` with no args, that powerless SA, no mounted token, `restartPolicy: Never`,
|
||||||
non-root with no privilege escalation and a read-only root filesystem, only the
|
non-root with no privilege escalation and a read-only root filesystem, only the
|
||||||
`workspace`/`tmp` **emptyDir** volumes, no init or ephemeral containers, only the
|
`workspace`/`tmp` **emptyDir** volumes, no init or ephemeral containers, only the
|
||||||
committed env **names**, and no `envFrom`.
|
committed env **names**, no `envFrom`, and exactly one `imagePullSecret` — the committed
|
||||||
|
one.
|
||||||
|
|
||||||
Six rules, because pinning `command` only moves the injection to whatever adjacent field
|
Seven rules, because pinning `command` only moves the injection to whatever adjacent field
|
||||||
nobody enumerated — on the sibling policy that was `initContainers` (measured: a Job with a
|
nobody enumerated — on the sibling policy that was `initContainers` (measured: a Job with a
|
||||||
compliant `containers[0]` and a privileged init container was **admitted**, pass 3 fail 0),
|
compliant `containers[0]` and a privileged init container was **admitted**, pass 3 fail 0),
|
||||||
then `ephemeralContainers`, then `env`, then `envFrom`. All of them are closed here from the
|
then `ephemeralContainers`, then `env`, then `envFrom`. All of them are closed here from the
|
||||||
start, and `platform/kyverno/tests/pm-agent/` proves each refusal offline: **28 assertions,
|
start, and `platform/kyverno/tests/pm-agent/` proves each refusal offline: **33 assertions,
|
||||||
28 passing** under the pinned `kyverno` CLI. The worker's own input guards have their own
|
33 passing** under the pinned `kyverno` CLI. The worker's own input guards have their own
|
||||||
offline probe — `infra/agent/probe-validation.sh`, **17 refusals** — because a guard nothing
|
offline probe — `infra/agent/probe-validation.sh`, **17 refusals** — because a guard nothing
|
||||||
exercises is a claim rather than a control.
|
exercises is a claim rather than a control.
|
||||||
|
|
||||||
@@ -230,6 +250,12 @@ five that matter:
|
|||||||
| `PM_MAX_ACTIVE_WORKERS` | `2` | concurrent workers. RAM is the binding constraint (2Gi limit each) and the subscription seat's rate limits are shared with every other use of that seat |
|
| `PM_MAX_ACTIVE_WORKERS` | `2` | concurrent workers. RAM is the binding constraint (2Gi limit each) and the subscription seat's rate limits are shared with every other use of that seat |
|
||||||
| `PM_MAX_NEW_WORKERS_PER_CYCLE` | `1` | drains a backlog one issue at a time instead of in one stampede the first minute the credentials work |
|
| `PM_MAX_NEW_WORKERS_PER_CYCLE` | `1` | drains a backlog one issue at a time instead of in one stampede the first minute the credentials work |
|
||||||
| `PM_PUSH_ON_RED_GATE` | `false` | a change the gate refused is not pushed; the issue gets the gate log instead |
|
| `PM_PUSH_ON_RED_GATE` | `false` | a change the gate refused is not pushed; the issue gets the gate log instead |
|
||||||
|
| `PM_DECISION_MAX_TURNS` | `1` (unset) | turns the triage call may take (1–5). The PM holds no tools, so a second turn is only ever the CLI re-asking after an answer that failed the decision schema. Raise it if a triage error logs `subtype='error_max_turns'` |
|
||||||
|
|
||||||
|
**When a triage fails**, the `[error] issue #N: claude exited …` line carries the CLI's result
|
||||||
|
envelope (`subtype`, `is_error`, `api_error_status`, `num_turns`, `result`) and the tails of
|
||||||
|
stdout and stderr, redacted. Before 2026-10-31 it logged stderr only, which is empty under
|
||||||
|
`--output-format json` — see `decisions/2026-10-31-pm-agent-triage-exit.md`.
|
||||||
|
|
||||||
Changing a knob requires bumping the `config-generation` annotation in `deployment.yaml` in
|
Changing a knob requires bumping the `config-generation` annotation in `deployment.yaml` in
|
||||||
the same commit — the process reads its configuration once, at startup, so without the bump
|
the same commit — the process reads its configuration once, at startup, so without the bump
|
||||||
|
|||||||
@@ -51,15 +51,20 @@ data:
|
|||||||
# the dispatcher and its workers share the committed Job template and the env-name
|
# the dispatcher and its workers share the committed Job template and the env-name
|
||||||
# contract. A tag here is refused at startup (and would be DENIED by admission anyway).
|
# contract. A tag here is refused at startup (and would be DENIED by admission anyway).
|
||||||
#
|
#
|
||||||
# PLACEHOLDER UNTIL THE IMAGE IS PUBLISHED. The all-zero digest below is deliberately
|
# THE FIRST PUBLISHED DIGEST. CI builds, verifies and — on main — publishes agent-runner
|
||||||
# impossible. CI builds, verifies and — on main — publishes agent-runner
|
# (.gitea/workflows/images.yaml). This value is images run 340's publish (tag
|
||||||
# (.gitea/workflows/images.yaml), and Renovate replaces this value with the digest the
|
# sha-523cd9118f95, which `main` pointed at), read from the registry on 2026-10-30. From here
|
||||||
# REGISTRY returned for the image's `main` tag, in a pull request that also moves the
|
# on Renovate replaces it with the digest the REGISTRY returns for the `main` tag, in a pull
|
||||||
# Deployment's image (same marker, same branch). Nobody pushes this image from a workstation.
|
# request that also moves the Deployment's image (same marker, same branch). Nobody pushes
|
||||||
# Until the first publish the dispatcher starts, fails to pull, and says so — which is the
|
# this image from a workstation.
|
||||||
# visible failure, not a silent one.
|
#
|
||||||
|
# The run-340 digest predated the worker template's `imagePullSecrets` entry (PR #61), which
|
||||||
|
# admission REQUIRES, so its worker Jobs were DENIED. The digest below (df36d9b, run 383,
|
||||||
|
# agent-runner:sha-ddd04d460ef6, pinned in PR #96) bakes a template that carries
|
||||||
|
# `imagePullSecrets: pm-agent-registry`, which is what made turning PM_DRY_RUN off possible
|
||||||
|
# (decisions/2026-10-31-pm-agent-runner-digest.md). A digest older than that would be DENIED.
|
||||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/agent-runner tag=main
|
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/agent-runner tag=main
|
||||||
PM_WORKER_IMAGE: "gitea.home.keithjackson.dev/home-network/agent-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
PM_WORKER_IMAGE: "gitea.home.keithjackson.dev/home-network/agent-runner@sha256:df36d9b7fef56d141220a387f64fb864cede8e5af30513ebfeb3ab566d3f771d"
|
||||||
|
|
||||||
# --- the model -------------------------------------------------------------
|
# --- the model -------------------------------------------------------------
|
||||||
# The PM only reads a thread and answers with one JSON object, so it holds no tools and
|
# The PM only reads a thread and answers with one JSON object, so it holds no tools and
|
||||||
@@ -104,12 +109,21 @@ data:
|
|||||||
# work somebody deliberately handed it while trust is being established, with no code
|
# work somebody deliberately handed it while trust is being established, with no code
|
||||||
# change and no redeploy of anything but this ConfigMap. The operator's call; the default
|
# change and no redeploy of anything but this ConfigMap. The operator's call; the default
|
||||||
# here honours the request as made.
|
# here honours the request as made.
|
||||||
PM_REQUIRE_LABEL: ""
|
#
|
||||||
|
# SET TO `agent` FOR THE FIRST RUN (2026-10-30, docs/plans/pm-agent-issue-to-pr.md
|
||||||
|
# follow-up 3). The label does not exist in the repository yet: create it in Gitea
|
||||||
|
# (Issues -> Labels -> New Label, name `agent`) before labelling a test issue.
|
||||||
|
PM_REQUIRE_LABEL: "agent"
|
||||||
|
|
||||||
# Dry run: the PM still reads threads, still asks the model and still posts its decision as
|
# Dry run: the PM still reads threads, still asks the model and still posts its decision as
|
||||||
# an issue comment — so the operator can read exactly what it WOULD have done — but creates
|
# an issue comment — so the operator can read exactly what it WOULD have done — but creates
|
||||||
# no worker Job and so changes no code. The honest way to watch this service for a day
|
# no worker Job and so changes no code. The honest way to watch this service for a day
|
||||||
# before letting it write anything.
|
# before letting it write anything.
|
||||||
|
#
|
||||||
|
# "true" for the first run (2026-10-30 .. 2026-10-31); dry-run triage of issue #97 was read
|
||||||
|
# and the operator approved going live. "false" since decisions/2026-10-31-pm-agent-go-live.md.
|
||||||
|
# Setting it back to "true" (plus a config-generation bump) is the gentlest stop — see
|
||||||
|
# runbooks/pm-agent.md.
|
||||||
PM_DRY_RUN: "false"
|
PM_DRY_RUN: "false"
|
||||||
|
|
||||||
# If the gate refuses a worker's change, the branch is NOT pushed by default and the issue
|
# If the gate refuses a worker's change, the branch is NOT pushed by default and the issue
|
||||||
|
|||||||
@@ -48,9 +48,13 @@ spec:
|
|||||||
# It is a hand-maintained marker rather than a computed hash because Flux's
|
# It is a hand-maintained marker rather than a computed hash because Flux's
|
||||||
# Kustomization here does not run a configmap generator; bump it in the same commit
|
# Kustomization here does not run a configmap generator; bump it in the same commit
|
||||||
# as a change to configmap.yaml.
|
# as a change to configmap.yaml.
|
||||||
pm-agent.homelab.keithjackson.dev/config-generation: "1"
|
pm-agent.homelab.keithjackson.dev/config-generation: "4"
|
||||||
spec:
|
spec:
|
||||||
serviceAccountName: pm-agent
|
serviceAccountName: pm-agent
|
||||||
|
# The kubelet's credential for pulling this image. Not the agent's Gitea credential —
|
||||||
|
# see pm-agent-registry.sops.yaml.example for why they are two Secrets of one token.
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: pm-agent-registry
|
||||||
# The dispatcher DOES need its token: `kubectl create job` is its one privileged act,
|
# The dispatcher DOES need its token: `kubectl create job` is its one privileged act,
|
||||||
# and platform/pm-agent/rbac.yaml is what bounds it to that. Contrast the worker, whose
|
# and platform/pm-agent/rbac.yaml is what bounds it to that. Contrast the worker, whose
|
||||||
# Job template sets this false because it needs no API access at all.
|
# Job template sets this false because it needs no API access at all.
|
||||||
@@ -69,10 +73,12 @@ spec:
|
|||||||
# disagree about the shape the admission policy enforces. Replaced together, in ONE
|
# disagree about the shape the admission policy enforces. Replaced together, in ONE
|
||||||
# Renovate pull request: both lines carry the same marker, so Renovate treats them as
|
# Renovate pull request: both lines carry the same marker, so Renovate treats them as
|
||||||
# one dependency and bumps them in the same branch. The digest comes from CI's publish
|
# one dependency and bumps them in the same branch. The digest comes from CI's publish
|
||||||
# of agent-runner's `main` tag (.gitea/workflows/images.yaml) — the all-zero
|
# of agent-runner's `main` tag (.gitea/workflows/images.yaml). The value below is the
|
||||||
# placeholder below is impossible on purpose (see configmap.yaml PM_WORKER_IMAGE).
|
# FIRST publish (images run 340, tag sha-523cd9118f95 = main at the time), read from
|
||||||
|
# the registry's manifest endpoint on 2026-10-30 and pinned by hand once so the
|
||||||
|
# dispatcher does not wait on Renovate's first run; every later bump is Renovate's.
|
||||||
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/agent-runner tag=main
|
# renovate-digest: image=gitea.home.keithjackson.dev/home-network/agent-runner tag=main
|
||||||
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
image: gitea.home.keithjackson.dev/home-network/agent-runner@sha256:df36d9b7fef56d141220a387f64fb864cede8e5af30513ebfeb3ab566d3f771d
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
command:
|
command:
|
||||||
- /usr/local/bin/pm-dispatch
|
- /usr/local/bin/pm-dispatch
|
||||||
|
|||||||
@@ -35,6 +35,12 @@ resources:
|
|||||||
# the distance between naming a file in a comment and listing it under `resources:` is one
|
# the distance between naming a file in a comment and listing it under `resources:` is one
|
||||||
# careless edit.
|
# careless edit.
|
||||||
- pm-agent-credentials.sops.yaml
|
- pm-agent-credentials.sops.yaml
|
||||||
|
# The KUBELET's pull credential for agent-runner — a separate Secret from the one above
|
||||||
|
# because it is a different consumer in a different format: dockerconfigjson, read before
|
||||||
|
# any container starts. Same underlying Gitea token, which must also carry `read:package`.
|
||||||
|
# Measured 2026-10-05: this registry refuses unauthenticated clients outright, so without
|
||||||
|
# this the pull 401s. Also placeholders until the operator fills it in.
|
||||||
|
- pm-agent-registry.sops.yaml
|
||||||
- deployment.yaml
|
- deployment.yaml
|
||||||
# default-deny baseline + the two egress allow policies (gitea, kube-apiserver for the
|
# default-deny baseline + the two egress allow policies (gitea, kube-apiserver for the
|
||||||
# dispatcher only, tcp/443 for the model). NO ingress allow exists, by design.
|
# dispatcher only, tcp/443 for the model). NO ingress allow exists, by design.
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ metadata:
|
|||||||
app.kubernetes.io/part-of: pm-agent
|
app.kubernetes.io/part-of: pm-agent
|
||||||
type: Opaque
|
type: Opaque
|
||||||
stringData:
|
stringData:
|
||||||
gitea-token: ENC[AES256_GCM,data:Vs6TkiNocU+qzvlv8mndXhtiqp1RyY00KhIBDuL8w0WoX72B8DxfPK8eWISy9gtx9TBlYpu2FHP7,iv:2z8+gzWuZDFACb06rnYg071h4wO7YlcTFNZDQzDxRNs=,tag:cUhj+uQUwu44HpPtFTVjVA==,type:str]
|
gitea-token: ENC[AES256_GCM,data:nkQT4m0DYykNcEUPV+1A4APuraekHN+8tY74HnRIbwU8SwYbRRqfKQ==,iv:d6Lj4Q/ys381msrrRtIIYbV9Ey3Nhua4zmX/7aCMlSk=,tag:xKKmc1i9TRuUVc/C2P1zHg==,type:str]
|
||||||
claude-code-oauth-token: ENC[AES256_GCM,data:2PjA5MOx8Enm4wxqYdJ3QY8juqPkXh4adQv3IRtw1Nb2Dw6SkOFc,iv:ZSlefJZqCN7g17D+27VkkVYu8F1umM7JOG+f/crZtsE=,tag:QyNYIbHr+ANS0Rx4USa0Iw==,type:str]
|
claude-code-oauth-token: ENC[AES256_GCM,data:Z5elUXr2wiu1S3FvSsHPNfhyrh+dGAdBf4HEmvg+Ky3QuCDpLv/XcMG5wWNywYVR4jlx/z2rm2FjjRYo+8rG0+I3SzriKG2MD7Zri4bDUf+DFksa6pI2oGDmgTjp8Zjz8oGapaMurRnuYcMm,iv:tXADrr2PD1zTpmTuaCA224wzbsq2RKzx0/ylU2k6suE=,tag:DMh31sAj0E/8ndcP1M6zAw==,type:str]
|
||||||
sops:
|
sops:
|
||||||
kms: []
|
kms: []
|
||||||
gcp_kms: []
|
gcp_kms: []
|
||||||
@@ -28,8 +28,8 @@ sops:
|
|||||||
V3d1UGVsTlljS3lHbFRkUWE4aVdRSDAK3vlChFXHg+jOJyjzUBZ4F4NqIn8rr5xl
|
V3d1UGVsTlljS3lHbFRkUWE4aVdRSDAK3vlChFXHg+jOJyjzUBZ4F4NqIn8rr5xl
|
||||||
WAH9bgY14yvX8qm3P4iLeFecNssqUvs2hvDW82vhMfUQlqYpTTXyjQ==
|
WAH9bgY14yvX8qm3P4iLeFecNssqUvs2hvDW82vhMfUQlqYpTTXyjQ==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
lastmodified: "2026-10-06T03:09:21Z"
|
lastmodified: "2026-10-31T16:07:12Z"
|
||||||
mac: ENC[AES256_GCM,data:lKj3CtZ3fCvOwuJSeNb5x8dj+5O0Uhfyoir+ZfIGD52xgeTWtmQ6CmOVbtrYjp1h1dgZnWopbibfMIKE9AmjSAHA7UJOen++zcy4qrA8Yh8xGzfhb1tOJplxZP+Lp9319VYQHqXqOScs7at8Osnw2RYuibW/j6eIY0B17JzIXvo=,iv:14aKtsMd9Q6QqnWVszoi+W2+z4QPfXUd7oZfK46+gRM=,tag:NwBg43/vNeFlgHmNLB7rYw==,type:str]
|
mac: ENC[AES256_GCM,data:rK1GR6FoIsh2Y6VSQSkLhGwf1U2kBrV3VYZT/5X+TxbkURza7DOTUyMJsY0oLKApvkTvOxLd3/WfzBNW/O6Tho1S4otvi4UC7fY6DjC46LmrfSlAxoRp7ov/oIRfwLBUanpNzfT94fu7sjoNM0lRlFp9iSpBU44upcIWwclK5WY=,iv:EujucxcPU+zfN9KmOhbqfVT5anv4g/jRg41hu5BgGx8=,tag:fViHjTT6w20b2AsXZodq0g==,type:str]
|
||||||
pgp: []
|
pgp: []
|
||||||
encrypted_regex: ^(data|stringData)$
|
encrypted_regex: ^(data|stringData)$
|
||||||
version: 3.9.4
|
version: 3.9.4
|
||||||
@@ -60,11 +60,21 @@
|
|||||||
# Keycloak service-account token therefore cannot authenticate to Gitea's API or to
|
# Keycloak service-account token therefore cannot authenticate to Gitea's API or to
|
||||||
# git-over-HTTP. Do not "upgrade" this to Keycloak without re-reading that.
|
# git-over-HTTP. Do not "upgrade" this to Keycloak without re-reading that.
|
||||||
#
|
#
|
||||||
# Scopes: `write:repository` and `write:issue`. Nothing else — in particular NOT
|
# Scopes, exactly these four (one token serves both Secrets — see
|
||||||
# `write:admin`, NOT `write:organization`, NOT `write:user`.
|
# pm-agent-registry.sops.yaml.example):
|
||||||
|
# read:user GET /user, the dispatcher's startup identity check. Gitea 1.24.3
|
||||||
|
# puts the whole /api/v1/user group behind the `user` scope
|
||||||
|
# category (routers/api/v1/api.go), so without it the dispatcher
|
||||||
|
# CrashLoops on its first call even with an otherwise-valid token.
|
||||||
|
# write:repository the clone, the branch push, the pull-request create, and the
|
||||||
|
# dispatcher's reads of pulls/branches/commits.
|
||||||
|
# write:issue reading open issues and posting issue comments.
|
||||||
|
# read:package the kubelet's image pull (pm-agent-registry).
|
||||||
|
# Nothing else — in particular NOT `write:admin`, NOT `write:organization`, NOT
|
||||||
|
# `write:user`. (Until 2026-10-30 this list said two scopes; `read:user` was missing
|
||||||
|
# and `read:package` lived only in the registry template.)
|
||||||
#
|
#
|
||||||
# `write:repository` covers the clone, the branch push and the pull-request create;
|
# (The repository's OTHER CI
|
||||||
# `write:issue` covers reading and posting issue comments. (The repository's OTHER CI
|
|
||||||
# token, `SETTINGS_API_TOKEN`, is a separate credential with `read:admin` — do not
|
# token, `SETTINGS_API_TOKEN`, is a separate credential with `read:admin` — do not
|
||||||
# conflate them, and do not give this service that one. The 2026-10-02 measurement on
|
# conflate them, and do not give this service that one. The 2026-10-02 measurement on
|
||||||
# PR #38 recorded in infra/gitea/approval-policy.json is about that token, not this one.)
|
# PR #38 recorded in infra/gitea/approval-policy.json is about that token, not this one.)
|
||||||
@@ -94,6 +104,17 @@
|
|||||||
#
|
#
|
||||||
# (4) COPY, FILL, ENCRYPT, COMMIT:
|
# (4) COPY, FILL, ENCRYPT, COMMIT:
|
||||||
#
|
#
|
||||||
|
# `sops` IS IN THE PINNED TOOLBOX, NOT ON THE HOST (measured 2026-10-30: absent from the
|
||||||
|
# workstation's PATH; sops 3.9.4 and vi in the toolbox). The simplest path needs no
|
||||||
|
# plaintext file at all — edit the committed, already-encrypted file in place, from a
|
||||||
|
# worktree, the way runbooks/vpn-remote-access.md does for its key:
|
||||||
|
#
|
||||||
|
# ./infra/devcontainer/dev.sh run sops platform/pm-agent/pm-agent-credentials.sops.yaml
|
||||||
|
#
|
||||||
|
# It opens vi on the decrypted document; replace the REPLACE-ME value(s), save, quit, and
|
||||||
|
# sops re-encrypts on write. If you take the cp route below instead, run the `sops -e -i`
|
||||||
|
# line through `./infra/devcontainer/dev.sh run` as well, and delete any plaintext temp file.
|
||||||
|
#
|
||||||
# cp platform/pm-agent/pm-agent-credentials.sops.yaml.example \
|
# cp platform/pm-agent/pm-agent-credentials.sops.yaml.example \
|
||||||
# platform/pm-agent/pm-agent-credentials.sops.yaml
|
# platform/pm-agent/pm-agent-credentials.sops.yaml
|
||||||
# $EDITOR platform/pm-agent/pm-agent-credentials.sops.yaml # the two real values
|
# $EDITOR platform/pm-agent/pm-agent-credentials.sops.yaml # the two real values
|
||||||
@@ -120,7 +141,7 @@ metadata:
|
|||||||
type: Opaque
|
type: Opaque
|
||||||
stringData:
|
stringData:
|
||||||
# Gitea token for the dedicated, NON-ADMIN `pm-agent` account.
|
# Gitea token for the dedicated, NON-ADMIN `pm-agent` account.
|
||||||
# Scopes: write:repository, write:issue.
|
# Scopes: read:user, write:repository, write:issue, read:package.
|
||||||
gitea-token: REPLACE-ME-gitea-token
|
gitea-token: REPLACE-ME-gitea-token
|
||||||
# Output of `claude setup-token` on a subscription account. One year's validity.
|
# Output of `claude setup-token` on a subscription account. One year's validity.
|
||||||
claude-code-oauth-token: REPLACE-ME-claude-code-oauth-token
|
claude-code-oauth-token: REPLACE-ME-claude-code-oauth-token
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
# PLACEHOLDER VALUES — the kubelet cannot pull agent-runner until the operator replaces them.
|
||||||
|
# Full hand-off steps: pm-agent-registry.sops.yaml.example (same directory).
|
||||||
|
# SOPS-encrypted to the repository's age recipient; keys stay cleartext for reviewable diffs.
|
||||||
|
#
|
||||||
|
# This is the KUBELET's pull credential, not the agent's Gitea credential. The agent's is
|
||||||
|
# pm-agent-credentials (Opaque, read as env by the application); this one is
|
||||||
|
# dockerconfigjson and is read before any container starts. Same underlying Gitea token,
|
||||||
|
# which must carry read:package as well as write:repository and write:issue.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: pm-agent-registry
|
||||||
|
namespace: pm-agent
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: pm-agent
|
||||||
|
app.kubernetes.io/part-of: pm-agent
|
||||||
|
type: kubernetes.io/dockerconfigjson
|
||||||
|
stringData:
|
||||||
|
.dockerconfigjson: ENC[AES256_GCM,data:NuKobTM0S0TK8U961uNTIIYiyAI2L3QCopZXvKxXHLC2tjpdyV+REKzYPmv+orCMiRsY8GNtfaHb8Hret4sQWHySAAKN0sEtL+OkzLyRH+u7irAGFYUBf8q+vIb+hnhdwejHfL1UEaHovJlZuIjqOdSItqlxqXURGtvUtBlkD7cnzDajzLyW+QTp5Gs3Fc8ps4XxRkv/589bKpn0DIEQiXUujydBZzKVnO5unxMea+4IBwcC6yaeQ+fXCoILrKD3PP2Igz0=,iv:vrQqH4dIUwDLSZGe1JKnsajtvUb7lJe3st/9K+4/w0M=,tag:P9VLwOqnTfM1uQnbousYqQ==,type:str]
|
||||||
|
sops:
|
||||||
|
kms: []
|
||||||
|
gcp_kms: []
|
||||||
|
azure_kv: []
|
||||||
|
hc_vault: []
|
||||||
|
age:
|
||||||
|
- recipient: age175l8mdpdfu7hq0aml3exs3gykarwjmuvr23sq56s3wjejnyk44wsp4mfzg
|
||||||
|
enc: |
|
||||||
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvdG1lQm16V0tJdWZXRXlU
|
||||||
|
Z2R4RXVyTXd6VTVJUWtkbUVjZldnb1JFOWhVCjVMMDdDb3AwVkt3TzJzdjR0VnNs
|
||||||
|
SFlncWg2cWlFTy9IZ1YxSC9Zd3ZMaEkKLS0tIEZFQWhIMW5zMURWYmlXaFg1ZjhD
|
||||||
|
cG1Fb0V3VncrTm9SQ091a0J0Z3RURWMKpdzNVub0zH5S8nnCSCt2H6NcCqDbV24L
|
||||||
|
Mf8K2eIfu/ixn2NfwoVVj2QO+n+3/P6PLphdoiKMrL+ms0+JqWFpkQ==
|
||||||
|
-----END AGE ENCRYPTED FILE-----
|
||||||
|
lastmodified: "2026-10-31T16:11:11Z"
|
||||||
|
mac: ENC[AES256_GCM,data:0GIYwtVcNngq2Zpz3haCx6L+j2SrgHFL5pRYeRM4Ac1T9kjKSwHik6Q4+T8FU+EJa9GP1yXrpdsjggsjzG9dI2zRyAF1C8x1znD5ls6OwT5jLak8QrHNcTavRlm6N6y1yOsQCTFMc+zeCywTnFJldTqPKH+6K/bNMNBuD14BNv8=,iv:oqGygkOmFgRw0TYbWO75/sJ/doXtorAWAg89LtaqIsg=,tag:vBO/DWV93ekZB86G+Hca2A==,type:str]
|
||||||
|
pgp: []
|
||||||
|
encrypted_regex: ^(data|stringData)$
|
||||||
|
version: 3.9.4
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (`.yaml.example` is ignored by kustomize and is
|
||||||
|
# never auto-encrypted by SOPS). Reference scaffold + OPERATOR HAND-OFF.
|
||||||
|
#
|
||||||
|
# WHAT THIS IS: the credential the KUBELET uses to pull `agent-runner` from the in-cluster
|
||||||
|
# Gitea registry. It is NOT the credential the agent uses to talk to Gitea — that one is
|
||||||
|
# `pm-agent-credentials` and it is read by the application. Two Secrets, because they are
|
||||||
|
# consumed by two different things in two different formats:
|
||||||
|
#
|
||||||
|
# pm-agent-credentials Opaque read by pm-dispatch / pm-worker as env vars
|
||||||
|
# pm-agent-registry dockerconfigjson read by the KUBELET, before any container runs
|
||||||
|
#
|
||||||
|
# WHY A PULL CREDENTIAL IS NEEDED AT ALL. Measured 2026-10-05: an unauthenticated client
|
||||||
|
# gets nothing from this Gitea's registry — it issues an anonymous token whose `access`
|
||||||
|
# claim is `null`, and every path answers 401 whether it exists or not
|
||||||
|
# (`/api/v1/packages/Home-Network` replies `{"message":"token is required"}`). So a pull
|
||||||
|
# with no credential cannot succeed, and before this Secret there was no `imagePullSecret`
|
||||||
|
# anywhere in the tree. See decisions/2026-10-05-pm-agent-registry-pull-prereq.md.
|
||||||
|
#
|
||||||
|
# THE OTHER HALF, ALREADY FIXED: the nodes used to resolve this registry's hostname to the
|
||||||
|
# non-hairpinned WAN address and fail TLS against the router's `CN=%D` placeholder
|
||||||
|
# certificate. That is the `/etc/hosts` pin in
|
||||||
|
# infra/ansible/inventory/group_vars/k3s_prod/vars.yml
|
||||||
|
# (decisions/2026-10-06-pm-agent-node-resolves-gitea-to-wan.md). Both halves are needed; a
|
||||||
|
# pull fails on whichever is missing.
|
||||||
|
#
|
||||||
|
# =============================================================================
|
||||||
|
# HAND-OFF STEPS (operator)
|
||||||
|
# =============================================================================
|
||||||
|
#
|
||||||
|
# (1) USE THE SAME `pm-agent` ACCOUNT the agent already has (created per
|
||||||
|
# pm-agent-credentials.sops.yaml.example — the dedicated, NON-ADMIN one). A second
|
||||||
|
# account would be a second thing to keep non-admin for no gain: this token reads one
|
||||||
|
# package and nothing else.
|
||||||
|
#
|
||||||
|
# (2) ADD THE PACKAGE SCOPE. Gitea token scopes are fixed at creation, so this means
|
||||||
|
# generating a token with all four scopes and using it in BOTH Secrets:
|
||||||
|
#
|
||||||
|
# read:package <- NEW, what the kubelet needs to pull
|
||||||
|
# read:user <- the dispatcher's startup GET /user (Gitea 1.24.3 scopes it)
|
||||||
|
# write:repository <- the agent's clone / branch push / pull-request create
|
||||||
|
# write:issue <- the agent's issue comments
|
||||||
|
#
|
||||||
|
# THE SCOPE IS NOT THE PERMISSION. `read:package` only lets the token use whatever
|
||||||
|
# package access the ACCOUNT has, and these packages belong to the ORGANISATION
|
||||||
|
# (`home-network/…`), not to a repository. Gitea 1.24.3 decides org-package access in
|
||||||
|
# services/context/package.go `determineAccessMode`: the highest Packages-unit mode of
|
||||||
|
# any org team the user is in, else READ if the org is visible to that user. Home-Network
|
||||||
|
# is `public` today (measured 2026-10-30), so a signed-in non-member already gets read —
|
||||||
|
# but that is a side effect of a visibility setting nobody chose for this. Make it
|
||||||
|
# explicit, and independent of visibility:
|
||||||
|
#
|
||||||
|
# Organisation Home-Network -> Teams -> New Team
|
||||||
|
# name: pm-agent-pull permission: none (unit-level)
|
||||||
|
# repositories: NONE (do not tick "all repositories")
|
||||||
|
# units: Packages = Read; every other unit = No Access
|
||||||
|
# then add `pm-agent` as its only member.
|
||||||
|
#
|
||||||
|
# That grants package read on the org and nothing on any repository; the repo `Write`
|
||||||
|
# comes only from the collaborator entry in the credentials template.
|
||||||
|
#
|
||||||
|
# Gitea -> (as pm-agent) Settings -> Applications -> Generate New Token.
|
||||||
|
# Put the SAME value in `pm-agent-credentials` (`gitea-token`) and in the
|
||||||
|
# `.dockerconfigjson` below. One token, two consumers, two formats — the same
|
||||||
|
# one-value-in-two-files pattern the Keycloak client secret already uses here, and for
|
||||||
|
# the same reason: a Secret cannot be shared across formats.
|
||||||
|
#
|
||||||
|
# (3) BUILD THE DOCKERCONFIGJSON. `auth` is base64 of `<username>:<token>` — NOT a hash,
|
||||||
|
# just encoding, so treat this file exactly as secret as the token itself:
|
||||||
|
#
|
||||||
|
# USER=pm-agent
|
||||||
|
# TOKEN='<the token from step 2>'
|
||||||
|
# AUTH=$(printf '%s:%s' "$USER" "$TOKEN" | base64 -w0)
|
||||||
|
# cat > /tmp/dockerconfig.json <<JSON
|
||||||
|
# {"auths":{"gitea.home.keithjackson.dev":{"username":"$USER","password":"$TOKEN","auth":"$AUTH"}}}
|
||||||
|
# JSON
|
||||||
|
#
|
||||||
|
# THE REGISTRY KEY MUST BE THE HOST AS THE IMAGE REFERENCE SPELLS IT —
|
||||||
|
# `gitea.home.keithjackson.dev`, no scheme, no port, no path. The kubelet matches the
|
||||||
|
# pull host against these keys literally; `https://…` or a trailing `/v2/` silently
|
||||||
|
# matches nothing and the pull is attempted anonymously, which 401s exactly as if this
|
||||||
|
# Secret were absent.
|
||||||
|
#
|
||||||
|
# (4) COPY, FILL, ENCRYPT, COMMIT:
|
||||||
|
#
|
||||||
|
# `sops` IS IN THE PINNED TOOLBOX, NOT ON THE HOST (measured 2026-10-30: absent from the
|
||||||
|
# workstation's PATH; sops 3.9.4 and vi in the toolbox). The simplest path needs no
|
||||||
|
# plaintext file at all — edit the committed, already-encrypted file in place, from a
|
||||||
|
# worktree, the way runbooks/vpn-remote-access.md does for its key:
|
||||||
|
#
|
||||||
|
# ./infra/devcontainer/dev.sh run sops platform/pm-agent/pm-agent-registry.sops.yaml
|
||||||
|
#
|
||||||
|
# It opens vi on the decrypted document; replace the REPLACE-ME value(s), save, quit, and
|
||||||
|
# sops re-encrypts on write. If you take the cp route below instead, run the `sops -e -i`
|
||||||
|
# line through `./infra/devcontainer/dev.sh run` as well, and delete any plaintext temp file.
|
||||||
|
#
|
||||||
|
# cp platform/pm-agent/pm-agent-registry.sops.yaml.example \
|
||||||
|
# platform/pm-agent/pm-agent-registry.sops.yaml
|
||||||
|
# # paste the one-line JSON from step 3 as the .dockerconfigjson value
|
||||||
|
# sops -e -i platform/pm-agent/pm-agent-registry.sops.yaml
|
||||||
|
#
|
||||||
|
# The root `.sops.yaml` rule matches this path and encrypts `stringData`, so the diff
|
||||||
|
# stays reviewable. NEVER `sops -d` it into a terminal or a transcript (CLAUDE.md §6).
|
||||||
|
# It is already listed under `resources:` in platform/pm-agent/kustomization.yaml.
|
||||||
|
#
|
||||||
|
# (5) ROTATION. Rotating the Gitea token means updating BOTH Secrets, because it is one
|
||||||
|
# token. The dispatcher must then be restarted to re-read its env; worker Jobs and the
|
||||||
|
# kubelet pick the new value up on their own.
|
||||||
|
# =============================================================================
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: pm-agent-registry
|
||||||
|
namespace: pm-agent
|
||||||
|
type: kubernetes.io/dockerconfigjson
|
||||||
|
stringData:
|
||||||
|
# One line. Registry key = the host exactly as the image reference spells it.
|
||||||
|
.dockerconfigjson: '{"auths":{"gitea.home.keithjackson.dev":{"username":"pm-agent","password":"REPLACE-ME-the-same-gitea-token","auth":"REPLACE-ME-base64-of-username:token"}}}'
|
||||||
+184
-5
@@ -149,6 +149,50 @@
|
|||||||
],
|
],
|
||||||
"datasourceTemplate": "docker",
|
"datasourceTemplate": "docker",
|
||||||
"versioningTemplate": "docker"
|
"versioningTemplate": "docker"
|
||||||
|
},
|
||||||
|
// ADDED 2026-10-30 — THE CATALOG IS THE SOURCE; apps/wave-*/*.yaml IS GENERATED FROM IT.
|
||||||
|
// First real dry run (dispatch runs 353/354): EVERY app-image update edited only the rendered
|
||||||
|
// apps/wave-*/*.yaml and NONE edited catalog/. The `kubernetes` manager lists catalog/ above
|
||||||
|
// but extracts nothing there: a ServiceDefinition has no `apiVersion`, so it is not a
|
||||||
|
// Kubernetes manifest to that manager (measured: 0 deps from catalog/). Every such PR would
|
||||||
|
// fail ci/render-golden.sh (the canonical render reverts the bump), sit red and hold a
|
||||||
|
// prConcurrentLimit slot. These two managers read the SOURCES, under the SAME depName as the
|
||||||
|
// rendered line, so both land in ONE branch and render==golden holds: `source.image` in each
|
||||||
|
// ServiceDefinition, and render.py's OAUTH2_PROXY_IMAGE (the six auth sidecars).
|
||||||
|
{
|
||||||
|
"customType": "regex",
|
||||||
|
"description": "ServiceDefinition `source.image: <repo>:<tag>` (catalog/*.yaml) — the source of the rendered apps/wave-* image.",
|
||||||
|
"managerFilePatterns": ["/(^|/)catalog/[^/]+\\.ya?ml$/"],
|
||||||
|
"matchStrings": [
|
||||||
|
"\\n\\s+image:\\s*(?<depName>[^\\s:@#\"']+):(?<currentValue>[^\\s@#\"']+)"
|
||||||
|
],
|
||||||
|
"datasourceTemplate": "docker"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"customType": "regex",
|
||||||
|
"description": "catalog/render.py OAUTH2_PROXY_IMAGE — the per-app oauth2-proxy sidecar every forward-auth app is rendered with.",
|
||||||
|
"managerFilePatterns": ["/(^|/)catalog/render\\.py$/"],
|
||||||
|
"matchStrings": [
|
||||||
|
"\\nOAUTH2_PROXY_IMAGE\\s*=\\s*\"(?<depName>[^\":@]+):(?<currentValue>[^\"@]+)\""
|
||||||
|
],
|
||||||
|
"datasourceTemplate": "docker"
|
||||||
|
},
|
||||||
|
// ADDED 2026-10-30 — CNPG `Cluster.spec.imageName`. The `kubernetes` manager reads `image:` only,
|
||||||
|
// so the three database Clusters (platform/keycloak/postgres-cluster.yaml, apps/wave-0/
|
||||||
|
// nextcloud-db.yaml, apps/wave-0/semaphore-db.yaml) were untracked, while the grant Job in
|
||||||
|
// nextcloud-db.yaml — whose comment says "keep the tag in step with spec.imageName" — WAS. A real
|
||||||
|
// run would have bumped the Job and left the database behind. Same depName, so one branch.
|
||||||
|
{
|
||||||
|
"customType": "regex",
|
||||||
|
"description": "CloudNativePG Cluster `imageName: <repo>:<tag>`.",
|
||||||
|
"managerFilePatterns": [
|
||||||
|
"/(^|/)platform/.+\\.ya?ml$/",
|
||||||
|
"/(^|/)apps/.+\\.ya?ml$/"
|
||||||
|
],
|
||||||
|
"matchStrings": [
|
||||||
|
"\\n\\s+imageName:\\s*(?<depName>[^\\s:@#\"']+):(?<currentValue>[^\\s@#\"']+)"
|
||||||
|
],
|
||||||
|
"datasourceTemplate": "docker"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"packageRules": [
|
"packageRules": [
|
||||||
@@ -200,7 +244,15 @@
|
|||||||
"longhorn",
|
"longhorn",
|
||||||
"cilium",
|
"cilium",
|
||||||
"cloudnative-pg",
|
"cloudnative-pg",
|
||||||
"kube-prometheus-stack"
|
"kube-prometheus-stack",
|
||||||
|
// ADDED 2026-10-30. gitea and act_runner ARE the merge gate: a bad auto-merged bump takes
|
||||||
|
// down the very check that would have to approve its revert (the dry run proposed gitea
|
||||||
|
// 1.24.3 -> 1.27.3 and act_runner 0.2.11 -> 0.6.1, both as auto-mergeable "minor"; a 0.x
|
||||||
|
// minor is a breaking line by semver). The postgresql image restarts every CNPG database,
|
||||||
|
// Keycloak's (the IdP) included, now that imageName is tracked.
|
||||||
|
"gitea/gitea",
|
||||||
|
"gitea/act_runner",
|
||||||
|
"ghcr.io/cloudnative-pg/postgresql"
|
||||||
],
|
],
|
||||||
"automerge": false,
|
"automerge": false,
|
||||||
"addLabels": [
|
"addLabels": [
|
||||||
@@ -208,6 +260,35 @@
|
|||||||
"blast-radius"
|
"blast-radius"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
// ADDED 2026-10-30 — THE KEYCLOAK OPERATOR IS THE KEYCLOAK VERSION. The operator, its CRDs
|
||||||
|
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
||||||
|
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
||||||
|
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
||||||
|
// 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||||
|
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
||||||
|
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
||||||
|
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
||||||
|
// Admin API shapes read from that tag's source (sub-flow rows named by displayName, the
|
||||||
|
// RpEntityName all-or-nothing guard, requiredActions REPLACE-not-merge, ...). The gate is
|
||||||
|
// static and does not run that reconciler against a server, so a green check says nothing
|
||||||
|
// about whether a bump breaks it. Majors still wait on the dashboard (rule at the bottom).
|
||||||
|
{
|
||||||
|
"description": "Keycloak operator + CRDs + server (lockstep, one branch). IdP blast radius; passkey-reconciler.yaml encodes 26.0.6 Admin API shapes — re-check it against the new tag before merging. Never auto-merge.",
|
||||||
|
"matchDepNames": ["keycloak/keycloak-k8s-resources"],
|
||||||
|
"groupName": "keycloak-operator",
|
||||||
|
"automerge": false,
|
||||||
|
"addLabels": ["needs-review", "blast-radius"]
|
||||||
|
},
|
||||||
|
// ADDED 2026-10-30 — quay.io/keycloak/keycloak IS NOT A DEPENDENCY RENOVATE MAY MOVE. Its only
|
||||||
|
// occurrences are catalog/keycloak.yaml `source.image` and the apps/wave-1/keycloak.yaml it
|
||||||
|
// renders, which is NOT listed in apps/wave-1/kustomization.yaml and deploys nothing (the
|
||||||
|
// operator above runs the server). A PR bumping it would change no running pod while looking
|
||||||
|
// like a Keycloak upgrade, and could drift from the operator's real version.
|
||||||
|
{
|
||||||
|
"description": "quay.io/keycloak/keycloak appears only in the documentation-only catalog/keycloak.yaml and its undeployed render; the operator (keycloak/keycloak-k8s-resources) chooses the real server image.",
|
||||||
|
"matchPackageNames": ["quay.io/keycloak/keycloak"],
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "ADDED 2026-10-29. This repo's own images are followed ONLY through the `# renovate-digest:` markers above. The `kubernetes` manager also sees platform/pm-agent/deployment.yaml and infra/agent/worker-job-template.yaml, where the image has a digest and no tag; left on, it would look those up as `latest` (a tag CI never pushes) and, for the template, re-open the self-referential loop the custom manager deliberately avoids.",
|
"description": "ADDED 2026-10-29. This repo's own images are followed ONLY through the `# renovate-digest:` markers above. The `kubernetes` manager also sees platform/pm-agent/deployment.yaml and infra/agent/worker-job-template.yaml, where the image has a digest and no tag; left on, it would look those up as `latest` (a tag CI never pushes) and, for the template, re-open the self-referential loop the custom manager deliberately avoids.",
|
||||||
"matchManagers": ["kubernetes"],
|
"matchManagers": ["kubernetes"],
|
||||||
@@ -227,6 +308,96 @@
|
|||||||
"gitea.home.keithjackson.dev/home-network/conformance-runner"
|
"gitea.home.keithjackson.dev/home-network/conformance-runner"
|
||||||
],
|
],
|
||||||
"addLabels": ["own-image"]
|
"addLabels": ["own-image"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. The `flux` manager also extracts `image:` lines inside HelmRelease values, which the `kubernetes` manager already owns. Measured: exactly two (alpine in platform/oauth2-proxy/helmrelease.yaml, velero-plugin-for-aws in platform/velero/helmrelease.yaml), each extracted twice; the second rewrite of the same line logged `Cannot find replaceString in current file content` in runs 353/354. Flux keeps its charts (datasource helm).",
|
||||||
|
"matchManagers": ["flux"],
|
||||||
|
"matchDatasources": ["docker"],
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. MIRRORS catalog/<svc>.yaml `update.range` — the operator's declared per-service policy (schemas/servicedefinition.schema.json), which Renovate does not read. Without it the dry run proposed gitea 28.2.0, qbittorrent 20.04.1 (both stale/odd tags, not real successors), radarr 6.x and nextcloud 35 (four majors past 31; Nextcloud cannot skip majors). KEEP IN STEP with the catalog ranges.",
|
||||||
|
"matchPackageNames": ["gitea/gitea"],
|
||||||
|
"allowedVersions": "<2.0.0"
|
||||||
|
},
|
||||||
|
{ "matchPackageNames": ["nextcloud"], "allowedVersions": "<32.0.0" },
|
||||||
|
{ "matchPackageNames": ["lscr.io/linuxserver/qbittorrent"], "allowedVersions": "<6.0.0" },
|
||||||
|
{ "matchPackageNames": ["lscr.io/linuxserver/radarr"], "allowedVersions": "<6.0.0" },
|
||||||
|
{ "matchPackageNames": ["lscr.io/linuxserver/sonarr"], "allowedVersions": "<5.0.0" },
|
||||||
|
{ "matchPackageNames": ["semaphoreui/semaphore"], "allowedVersions": "<3.0.0" },
|
||||||
|
// ADDED 2026-10-30 — pihole, jackett and plex, PINNED to the digests that were already running
|
||||||
|
// (catalog/{pihole,jackett,plex}.yaml; each tag was resolved against the registry and matched the
|
||||||
|
// pods' imageID, so the pin changed nothing at runtime). None of the three catalog `update.range`
|
||||||
|
// values has a ceiling (pihole >=2024.0.0, jackett >=0.22.0, plex >=1.40.0), so the
|
||||||
|
// allowedVersions mirror above needs no entry for them.
|
||||||
|
//
|
||||||
|
// pihole/pihole needs NO rule: its CalVer tags (YYYY.MM.N, e.g. 2026.10.0) parse under the
|
||||||
|
// default `docker` versioning. Patches and minors (2026.10.0 -> 2026.10.1 / 2026.11.0) therefore
|
||||||
|
// AUTO-MERGE on a green gate under rule 1 — the operator's explicit choice, not an oversight; do
|
||||||
|
// not add a hold here. A YEAR ROLLOVER (2026.x -> 2027.x) is a MAJOR, so it waits for a tick on
|
||||||
|
// the Dependency Dashboard (rule below) and is never auto-merged.
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. LinuxServer tags: `vX.Y.Z-lsN` is the IMMUTABLE build tag (bare `X.Y.Z` and `latest` are re-pushed for every ls rebuild). Docker versioning would treat `-ls7` as a compatibility suffix and only ever compare tags with the SAME suffix, so it would never move. This is Renovate's own regex-versioning example for linuxserver images: the ls build number becomes the `build` component, so an ls rebuild of the same Jackett is an update too. The `^v` and `$` anchors exclude `latest`, `development`, the bare mutable `X.Y.Z` tags and the arch-prefixed (`amd64-v...`) tags.",
|
||||||
|
"matchDepNames": ["lscr.io/linuxserver/jackett"],
|
||||||
|
"versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-ls(?<build>\\d+)$"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. LOOK JACKETT UP ON ghcr.io, NOT lscr.io. lscr.io is LinuxServer's redirector in front of ghcr.io/linuxserver (same manifests: v0.24.2865-ls7 is sha256:4fcf3cc2... on both). Renovate's docker datasource fetches up to 1000 pages of tags from ghcr.io but only `dockerMaxPages` (default 20, a GLOBAL-only option) from any other host (lib/modules/datasource/docker/index.js getTags). Jackett has 32,093 tags at ~1000 per page (measured 2026-10-30), oldest first; the running tag is #31,980, so a lookup via lscr.io never sees it or anything newer and silently proposes NOTHING (measured in a dry run before this rule). overridePackageName changes only the lookup; the file keeps `lscr.io/...`, so the deployed image string does not change. Matched by depName because packageName is what this rule rewrites.",
|
||||||
|
"matchDepNames": ["lscr.io/linuxserver/jackett"],
|
||||||
|
"overridePackageName": "ghcr.io/linuxserver/jackett"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. Jackett publishes ~daily (13 ls builds in 12 days, measured 2026-10-30) and every one is a `patch` under the regex above, i.e. auto-mergeable. Left on the global daily window that is a PR, a gate run and a jackett pod restart EVERY DAY for an indexer proxy whose individual builds nobody needs promptly. WEEKLY instead: Renovate proposes only the newest tag when the window opens, so nothing is skipped, it is just batched. The window is still inside the global 03:00-06:00 local one that .gitea/workflows/renovate.yaml's 08:15 UTC cron lands in. NOTE: catalog/jackett.yaml `update.cadence` still says daily (`0 4 * * *`); that field drove Flux image-automation, which is retired, and this rule is the effective cadence now.",
|
||||||
|
"matchDepNames": ["lscr.io/linuxserver/jackett"],
|
||||||
|
"schedule": ["after 3am and before 6am on sunday"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. Plex tags are `<major>.<minor>.<patch>.<build>-<commit hex>` (e.g. 1.43.4.10903-e5521bd8c). Docker versioning reads `-e5521bd8c` as a compatibility suffix that differs on EVERY release, so it would never find a successor. The trailing `$` excludes the per-arch variants and the floating `latest`/`public`/`beta`/`plexpass` tags. A version tag does NOT self-update inside the container: cont-init 50-plex-update exits early when /version.txt matches the installed package, so what is pinned is what runs.",
|
||||||
|
"matchPackageNames": ["plexinc/pms-docker"],
|
||||||
|
"versioning": "regex:^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)\\.(?<build>\\d+)-[0-9a-f]+$"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. A CNPG Postgres MAJOR is a migration (new cluster + import), not an image bump; the dry run proposed 16.4 -> 18.6. Minors stay (and are blast-radius, above).",
|
||||||
|
"matchPackageNames": ["ghcr.io/cloudnative-pg/postgresql"],
|
||||||
|
"allowedVersions": "<17.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. A HelmRelease semver RANGE is a policy ceiling, and Renovate does not respect it — it WIDENS it (dry run: external-dns `<1.22.0` -> `<1.24.0`, headlamp `<0.42.0` -> `<0.47.0`), and the default rule would then auto-merge that as a 'minor', silently letting Flux adopt every new minor. Raising a ceiling is deliberate per-component work: propose it, never auto-merge it.",
|
||||||
|
"matchManagers": ["flux"],
|
||||||
|
"matchCurrentValue": "/^[<>=~^]|\\.x$/",
|
||||||
|
"automerge": false,
|
||||||
|
"addLabels": ["needs-review", "range-widen"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. velero-plugin-for-aws is pinned to a RELEASE CANDIDATE (v1.14.3-rc.2) and Renovate proposed NOTHING for it, although v1.14.3 and v1.14.4 are stable on Docker Hub. Docker versioning treats everything after the first `-` as a compatibility SUFFIX, so only other `*-rc.2` tags qualify and the RC is a fossil forever. Semver versioning reads it as a prerelease, and a prerelease upgrades to the stable release.",
|
||||||
|
"matchPackageNames": ["velero/velero-plugin-for-aws"],
|
||||||
|
"versioning": "semver"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. A python MINOR is a language-runtime release (the dry run proposed 3.13 -> 3.15 for the Keycloak reconcilers), and the gate does not execute those scripts. Review it; patches still auto-merge.",
|
||||||
|
"matchPackageNames": ["python"],
|
||||||
|
"matchUpdateTypes": ["minor"],
|
||||||
|
"automerge": false,
|
||||||
|
"addLabels": ["needs-review"]
|
||||||
|
},
|
||||||
|
// OPERATOR DECISION 2026-10-30: these two never auto-merge, at ANY update type. Both sit after
|
||||||
|
// the auto-merge rule (rule 1) so they override it; `needs-review` is added, not replaced.
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30 (operator decision). The oauth2-proxy image, under the ONE depName every source of it emits: catalog/render.py OAUTH2_PROXY_IMAGE (custom regex manager), the rendered apps/wave-* sidecars and platform/crafty/deployment.yaml (kubernetes manager). WHY: since v7.13.0 oauth2-proxy validates REFRESHED OIDC sessions against the ACCESS token instead of the id_token (upstream CHANGELOG, v7.13.0), every sidecar sets --cookie-refresh, and that refresh path has never run against this Keycloak (the platform oauth2-proxy runs v7.15.5 WITHOUT --cookie-refresh). render-golden and render.py --selftest pass on the bump, but both check rendered text, not the binary, so a green gate is not evidence here.",
|
||||||
|
"matchPackageNames": ["quay.io/oauth2-proxy/oauth2-proxy"],
|
||||||
|
"automerge": false,
|
||||||
|
"addLabels": ["needs-review"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30 (operator decision). Renovate's OWN version: the job `container.image` pin in .gitea/workflows/renovate.yaml (ghcr.io/renovatebot/renovate), and renovatebot/github-action should it ever be reintroduced (ci/lint.sh rejects it today). WHY: nothing re-validates renovate.json5 against the NEW version before it takes effect. The gate does not run renovate-config-validator, and this exact gap has bitten before: the action's default 39.x rejected this file (managerFilePatterns needs >= 40.2.0) and every run stopped at CONFIG_VALIDATION. Run the validator from the new tag, then merge.",
|
||||||
|
"matchPackageNames": ["ghcr.io/renovatebot/renovate", "renovatebot/github-action"],
|
||||||
|
"automerge": false,
|
||||||
|
"addLabels": ["needs-review"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-10-30. MAJORS WAIT FOR A TICK ON THE DEPENDENCY DASHBOARD instead of opening a PR. They are never auto-merged anyway, and with prConcurrentLimit 5 the dry run's 10 majors would hold every PR slot open indefinitely, starving the patch/minor/digest PRs that can merge on green.",
|
||||||
|
"matchUpdateTypes": ["major"],
|
||||||
|
"dependencyDashboardApproval": true
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
// WHAT RENOVATE STILL CANNOT FIX, so nobody mistakes a working bot for full coverage. A
|
// WHAT RENOVATE STILL CANNOT FIX, so nobody mistakes a working bot for full coverage. A
|
||||||
@@ -235,10 +406,18 @@
|
|||||||
// lscr.io/linuxserver/jackett:latest, pihole/pihole (NO TAG AT ALL), quay.io/keycloak/keycloak
|
// lscr.io/linuxserver/jackett:latest, pihole/pihole (NO TAG AT ALL), quay.io/keycloak/keycloak
|
||||||
// (no tag), nextcloud:31-apache and docker:27-dind. Those must be pinned BY HAND to a concrete
|
// (no tag), nextcloud:31-apache and docker:27-dind. Those must be pinned BY HAND to a concrete
|
||||||
// version before any automation can maintain them, and until then they drift on every pull with
|
// version before any automation can maintain them, and until then they drift on every pull with
|
||||||
// nothing recording what is running. Separately, the HelmRelease minor-locking ranges (9 of 12,
|
// nothing recording what is running. CORRECTED 2026-10-30: nextcloud:31-apache and docker:27-dind
|
||||||
// e.g. `>=88.0.0 <89.0.0` on kube-prometheus-stack) cap Renovate at patches inside a line it
|
// ARE maintained (the first dry run proposed 35-apache and 29-dind; the suffix is kept).
|
||||||
// has been told not to leave; raising those ceilings is deliberate per-component work and is
|
// CORRECTED 2026-10-30: quay.io/keycloak/keycloak was never a running reference — the operator
|
||||||
// NOT part of enabling the bot. Both lists are in decisions/2026-10-17-automated-upgrades.md.
|
// picks the server image — and the Keycloak version is now tracked via the operator URLs.
|
||||||
|
// CORRECTED 2026-10-30 (feat/pin-pihole-jackett-plex): pms-docker, jackett and pihole are now
|
||||||
|
// PINNED (1.43.4.10903-e5521bd8c, v0.24.2865-ls7, 2026.10.0 — the digests that were running) and
|
||||||
|
// maintained via the versioning rules above. None of the three publishes "only latest": all three
|
||||||
|
// publish immutable version tags, they were simply never referenced.
|
||||||
|
// Separately, the HelmRelease minor-locking ranges (9 of 12, e.g. `>=88.0.0 <89.0.0` on
|
||||||
|
// kube-prometheus-stack): THIS FILE USED TO SAY they cap Renovate at patches. They do not —
|
||||||
|
// Renovate WIDENS the range (runs 353/354). The `range-widen` rule above keeps that a reviewed
|
||||||
|
// PR rather than an auto-merge. Both lists are in decisions/2026-10-17-automated-upgrades.md.
|
||||||
// platform/kyverno/tests/** ADDED 2026-10-17: those Kyverno policy fixtures pin `alpine:latest`
|
// platform/kyverno/tests/** ADDED 2026-10-17: those Kyverno policy fixtures pin `alpine:latest`
|
||||||
// and a `conformance-runner:latest`, which are test inputs whose whole point is to be a fixed
|
// and a `conformance-runner:latest`, which are test inputs whose whole point is to be a fixed
|
||||||
// shape, not software to keep current. `conformance/**` was already ignored for the same
|
// shape, not software to keep current. `conformance/**` was already ignored for the same
|
||||||
|
|||||||
@@ -112,10 +112,12 @@ A private repository with no `secretRef`, or an expired token. The message names
|
|||||||
### 4. The policy's range matches nothing
|
### 4. The policy's range matches nothing
|
||||||
`ImagePolicy` semver ranges come from each ServiceDefinition's `update.range` (for example
|
`ImagePolicy` semver ranges come from each ServiceDefinition's `update.range` (for example
|
||||||
`">=2024.0.0"` for pihole, `">=24.0.0"` for keycloak). A range that no published tag satisfies — or a
|
`">=2024.0.0"` for pihole, `">=24.0.0"` for keycloak). A range that no published tag satisfies — or a
|
||||||
repository that publishes only `latest` (this platform has `lscr.io/linuxserver/jackett:latest` and
|
repository that publishes only `latest` — leaves the policy with nothing to select. Remediation: fix
|
||||||
`plexinc/pms-docker:latest`) — leaves the policy with nothing to select. Remediation: fix the range in
|
the range in `catalog/<service>.yaml` and re-render; a `latest`-only image cannot participate in
|
||||||
`catalog/<service>.yaml` and re-render; a `latest`-only image cannot participate in automation at all
|
automation at all and should not carry a policy. (CORRECTED 2026-10-30: this page used to name
|
||||||
and should not carry a policy.
|
`lscr.io/linuxserver/jackett` and `plexinc/pms-docker` as `latest`-only. Neither is — both publish
|
||||||
|
immutable version tags (`v0.24.2865-ls7`, `1.43.4.10903-e5521bd8c`) — they were merely *deployed* as
|
||||||
|
`:latest`. Both are now pinned to those tags and maintained by Renovate, not by an ImagePolicy.)
|
||||||
|
|
||||||
### 5. Egress is blocked for that namespace
|
### 5. Egress is blocked for that namespace
|
||||||
Default-deny everywhere (R-NET-01) means outbound registry access is an explicit allow. Cross-read
|
Default-deny everywhere (R-NET-01) means outbound registry access is an explicit allow. Cross-read
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ which is why this page exists: the loop is scripted and the reconcile is **force
|
|||||||
waited for. Measured end to end on 2026-09-29: **~2m45s**.
|
waited for. Measured end to end on 2026-09-29: **~2m45s**.
|
||||||
|
|
||||||
**An expired or wrong token does not break anything.** Verified against the image's own first-run
|
**An expired or wrong token does not break anything.** Verified against the image's own first-run
|
||||||
script (and the copy inside `plexinc/pms-docker:latest`): a failed exchange gets a 403 with an empty
|
script (and the copy inside the image then running as `plexinc/pms-docker:latest`; pinned to a version
|
||||||
|
tag since 2026-10-30): a failed exchange gets a 403 with an empty
|
||||||
body, `curl`'s status is never checked, the token `sed` yields empty, the `if` does not fire, there is
|
body, `curl`'s status is never checked, the token `sed` yields empty, the `if` does not fire, there is
|
||||||
no `else` and no `exit 1`, and the script (no `set -e`) exits 0. The pod comes up **unclaimed and
|
no `else` and no `exit 1`, and the script (no `set -e`) exits 0. The pod comes up **unclaimed and
|
||||||
serving**. So a burnt token costs you one more attempt, not an outage — and the repo deliberately
|
serving**. So a burnt token costs you one more attempt, not an outage — and the repo deliberately
|
||||||
@@ -186,4 +187,5 @@ When the claim does not take, in order of how often it is the answer here:
|
|||||||
Source of truth: `runbooks/plex-claim.md`. Index: `/runbooks/`
|
Source of truth: `runbooks/plex-claim.md`. Index: `/runbooks/`
|
||||||
Upstream behaviour cited here is from `plexinc/pms-docker` branch `master`,
|
Upstream behaviour cited here is from `plexinc/pms-docker` branch `master`,
|
||||||
`root/etc/cont-init.d/40-plex-first-run` (s6-overlay v2), cross-checked against the same file inside
|
`root/etc/cont-init.d/40-plex-first-run` (s6-overlay v2), cross-checked against the same file inside
|
||||||
the running `plexinc/pms-docker:latest` image.
|
the image that was running at the time as `plexinc/pms-docker:latest` (the deployment has been pinned
|
||||||
|
to a version tag, `catalog/plex.yaml` `source.image`, since 2026-10-30).
|
||||||
@@ -371,8 +371,8 @@ verify".)
|
|||||||
2026-10-01, so this means a node changed — a rebuild, a BIOS reset, or a driver regression.
|
2026-10-01, so this means a node changed — a rebuild, a BIOS reset, or a driver regression.
|
||||||
**Fix the node.**
|
**Fix the node.**
|
||||||
|
|
||||||
4. **The image changed under you.** See "The floating-image caveat". A `:latest` pull can remove the
|
4. **The image changed under you.** See "The image-version caveat". A pms-docker version bump (a
|
||||||
mechanism this feature leans on with no commit in this repo.
|
Renovate PR, possibly auto-merged) can remove the mechanism this feature leans on.
|
||||||
|
|
||||||
5. **The session was never transcoding.** Direct Play / Direct Stream never reaches the transcoder,
|
5. **The session was never transcoding.** Direct Play / Direct Stream never reaches the transcoder,
|
||||||
so no `hardware transcoding:` line is written for it at all. An *absent* line is this, not a
|
so no `hardware transcoding:` line is written for it at all. An *absent* line is this, not a
|
||||||
@@ -446,17 +446,21 @@ revert you):
|
|||||||
Re-adding the field later is the same edit in reverse. Nothing about the Plex-side toggle or the
|
Re-adding the field later is the same edit in reverse. Nothing about the Plex-side toggle or the
|
||||||
`/config` PVC is affected either way — do **not** delete the PVC as part of this.
|
`/config` PVC is affected either way — do **not** delete the PVC as part of this.
|
||||||
|
|
||||||
## The floating-image caveat
|
## The image-version caveat
|
||||||
|
|
||||||
Everything this page states about the image's behaviour — the `cont-init.d` hook that adds the app
|
Everything this page states about the image's behaviour — the `cont-init.d` hook that adds the app
|
||||||
user to the device's group, the absence of `PLEX_PREFERENCE*`, the six `setPref` keys, the
|
user to the device's group, the absence of `PLEX_PREFERENCE*`, the six `setPref` keys, the
|
||||||
`HardwareAcceleratedCodecs` string — was read from digest **`sha256:e0ab27395614…`**
|
`HardwareAcceleratedCodecs` string — was read from digest **`sha256:e0ab27395614…`**
|
||||||
(`plexmediaserver 1.43.4.10903`).
|
(`plexmediaserver 1.43.4.10903`).
|
||||||
|
|
||||||
The deployment runs **`plexinc/pms-docker:latest` with `imagePullPolicy: Always`**. A silent pull can
|
**Since 2026-10-30 that digest is pinned:** `catalog/plex.yaml` names
|
||||||
therefore change the behaviour this feature depends on **with no commit in this repo**, while the
|
`plexinc/pms-docker:1.43.4.10903-e5521bd8c`, which resolved to exactly `sha256:e0ab27395614…` on
|
||||||
manifest still looks identical. Symptom to watch for: hardware transcoding that worked and quietly
|
Docker Hub. Before that the deployment ran the floating `:latest` (implicitly `imagePullPolicy:
|
||||||
stopped, with no git change to point at.
|
Always`), and a silent pull could change this behaviour with no commit in the repo. That is no longer
|
||||||
|
the failure mode: the image now changes only through a **Renovate PR** bumping that tag (patch/minor
|
||||||
|
ones auto-merge on a green gate, and the gate does not exercise QuickSync). So the symptom to watch
|
||||||
|
for is hardware transcoding that quietly stopped **after a `chore(deps)` pms-docker bump** — check
|
||||||
|
`git log -- catalog/plex.yaml` first.
|
||||||
|
|
||||||
```
|
```
|
||||||
infra/devcontainer/dev.sh run bash -c \
|
infra/devcontainer/dev.sh run bash -c \
|
||||||
|
|||||||
@@ -67,9 +67,11 @@ Work through these in order; the first three are by far the most likely.
|
|||||||
| Symptom in the log | Cause | Fix |
|
| Symptom in the log | Cause | Fix |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `GET /user returned no login — is GITEA_TOKEN valid?` and a CrashLoop | the credentials are still the committed **placeholders**, or the token was revoked | fill them in per `platform/pm-agent/pm-agent-credentials.sops.yaml.example` |
|
| `GET /user returned no login — is GITEA_TOKEN valid?` and a CrashLoop | the credentials are still the committed **placeholders**, or the token was revoked | fill them in per `platform/pm-agent/pm-agent-credentials.sops.yaml.example` |
|
||||||
| `ImagePullBackOff`, digest `sha256:0000…` | the image has never been published; the committed digest is the deliberate placeholder | CI publishes it, not a person: check the latest `images` workflow run on `main` (it fails loudly if the `REGISTRY_PUSH_TOKEN` secret is missing; re-run it with `workflow_dispatch`), then merge Renovate's digest-bump PR, which updates **both** `deployment.yaml` and `configmap.yaml` (`PM_WORKER_IMAGE`) together — but read the row below first |
|
| `ImagePullBackOff`, digest `sha256:0000…` | (historical: before 2026-10-30 the committed digest was an all-zero placeholder) the image has never been published at the digest named | CI publishes it, not a person: check the latest `images` workflow run on `main` (it fails loudly if the `REGISTRY_PUSH_TOKEN` secret is missing; re-run it with `workflow_dispatch`), then merge Renovate's digest-bump PR, which updates **both** `deployment.yaml` and `configmap.yaml` (`PM_WORKER_IMAGE`) together — but read the row below first |
|
||||||
| `ImagePullBackOff` with `x509: certificate is not valid for any names` | **diagnosed.** containerd pulls as a node process, resolves through the node's public upstreams, and gets the CNAME to the WAN IP — which this router does not hairpin, so it answers :443 with its own `CN=%D` placeholder certificate (no SANs at all). Nothing to do with the digest, the Gateway or the wildcard cert | the `/etc/hosts` pin for `gitea.{{ k3s_base_domain }}` → `192.168.2.20` in `infra/ansible/inventory/group_vars/k3s_prod/vars.yml` fixes it — the same pin the Keycloak issuer already has. Apply it with the project's own tooling, scoped to just that task: `cd infra/ansible && ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags hosts`. Full evidence: `decisions/2026-10-06-pm-agent-node-resolves-gitea-to-wan.md` |
|
| `ImagePullBackOff` with `x509: certificate is not valid for any names` | **diagnosed.** containerd pulls as a node process, resolves through the node's public upstreams, and gets the CNAME to the WAN IP — which this router does not hairpin, so it answers :443 with its own `CN=%D` placeholder certificate (no SANs at all). Nothing to do with the digest, the Gateway or the wildcard cert | the `/etc/hosts` pin for `gitea.{{ k3s_base_domain }}` → `192.168.2.20` in `infra/ansible/inventory/group_vars/k3s_prod/vars.yml` fixes it — the same pin the Keycloak issuer already has. Apply it with the project's own tooling, scoped to just that task: `cd infra/ansible && ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags hosts`. Full evidence: `decisions/2026-10-06-pm-agent-node-resolves-gitea-to-wan.md` |
|
||||||
| `ImagePullBackOff` with `401 Unauthorized` once TLS works | the pull is unauthenticated. An anonymous client gets nothing from this Gitea's registry, and no `imagePullSecret` exists anywhere in the tree | add one (a `dockerconfigjson`, SOPS-encrypted like `pm-agent-credentials`, referenced from the Deployment **and** the committed worker Job template) or allow anonymous reads of the package in Gitea — a posture decision, so yours |
|
| `ImagePullBackOff` with `401 Unauthorized` / `unauthorized` | the pull credential is wrong, absent or scoped without `read:package`. The `pm-agent-registry` Secret exists and is referenced; its *value* is what starts as a placeholder | check the key names only, never values: `kubectl -n pm-agent get secret pm-agent-registry -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}'` should list `.dockerconfigjson`. Then confirm the token carries `read:package`, that its account can read the org's packages (a team with the Packages unit — see the `.example`), and that the registry key inside the JSON is the bare host `gitea.home.keithjackson.dev` — with a scheme or a port it matches nothing and the pull goes out anonymously, which looks identical to a missing Secret. Steps: `platform/pm-agent/pm-agent-registry.sops.yaml.example` |
|
||||||
|
| `GET /user` answers `403` / `token does not have at least one of required scope(s): [read:user]` at startup, then a CrashLoop | the token lacks `read:user` — Gitea 1.24.3 scopes the whole `/api/v1/user` group, and the dispatcher's first call is `GET /user` | mint a new token with all four scopes (`read:user`, `write:repository`, `write:issue`, `read:package`) and put it in **both** Secrets |
|
||||||
|
| a worker Job is refused by admission (`pm-agent-job-shape`, `imagePullSecrets`) | `PM_WORKER_IMAGE` is a digest built before the template carried `imagePullSecrets` (PR #61), so its baked template lacks the entry admission requires | pin the agent-runner digest CI published after #61 merged (Renovate's bump PR); until then keep `PM_DRY_RUN=true` |
|
||||||
| `PM_WORKER_IMAGE must be digest-pinned` | someone put a tag in the ConfigMap | restore the digest form; the digest comes from CI's publish (`.gitea/workflows/images.yaml` job summary) and Renovate's bump PR, never from a hand push |
|
| `PM_WORKER_IMAGE must be digest-pinned` | someone put a tag in the ConfigMap | restore the digest form; the digest comes from CI's publish (`.gitea/workflows/images.yaml` job summary) and Renovate's bump PR, never from a hand push |
|
||||||
| `scope: ONLY issues labelled 'agent'` | `PM_REQUIRE_LABEL` is set; unlabelled issues are out of scope by configuration | label the issue, or clear the knob |
|
| `scope: ONLY issues labelled 'agent'` | `PM_REQUIRE_LABEL` is set; unlabelled issues are out of scope by configuration | label the issue, or clear the knob |
|
||||||
| `cycle: 2 worker(s) active` and nothing new | at `PM_MAX_ACTIVE_WORKERS` | wait, or raise it (RAM is the binding constraint: each worker's limit is 2Gi) |
|
| `cycle: 2 worker(s) active` and nothing new | at `PM_MAX_ACTIVE_WORKERS` | wait, or raise it (RAM is the binding constraint: each worker's limit is 2Gi) |
|
||||||
|
|||||||
Reference in new issue
Block a user