Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89f48e4a5f | ||
|
|
a775082ae1 | ||
|
|
1a33814522 | ||
|
|
4a3f6dee59 | ||
|
|
759a0e51c8 | ||
|
|
ea30ba876c | ||
|
|
e8d26669a4 | ||
|
|
048254bfaf | ||
|
|
dd94101647 | ||
|
|
60e3a24652 | ||
|
|
2d32b9bd7d | ||
|
|
9f82b2accf | ||
|
|
1b2ec82a9d | ||
|
|
3724d3068b | ||
|
|
0605ecf593 |
No files matched your search
@@ -80,7 +80,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # full history so ci/touched-requirements.sh can diff vs main
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
timeout-minutes: 75
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # the per-image change detection diffs against the base commit
|
||||
|
||||
|
||||
@@ -60,9 +60,9 @@ jobs:
|
||||
spec-immutability:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 # the implementation repo
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # the implementation repo
|
||||
- name: Check out the read-only spec (sibling path)
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
repository: ${{ vars.SPEC_REPO_SLUG }} # e.g. Home-Network/home-network-k3s (set in Gitea)
|
||||
path: ../home-network-k3s
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [spec-immutability]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 } # full history so touched-requirement diff works
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh lint
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l0
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [l0]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [render-golden]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
run: make -C ci l1
|
||||
- name: Upload L1 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: l1-junit-coverage
|
||||
path: |
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
needs: [l1]
|
||||
if: ${{ vars.CLUSTER_AVAILABLE == 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh conformance
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
run: make -C ci conformance
|
||||
- name: Upload L2/L3 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: conformance-junit-coverage
|
||||
path: |
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
needs: [lint, l0, render-golden, l1, conformance]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Decide
|
||||
env:
|
||||
R_LINT: ${{ needs.lint.result }}
|
||||
|
||||
+38
-2
@@ -25,8 +25,44 @@ if [ "${#files[@]}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${files[@]}"
|
||||
ok "yaml/lint passed (${#files[@]} files)"
|
||||
# yamllint is single-threaded, so the file list is split into LINT_JOBS CONTIGUOUS slices of the
|
||||
# sorted list, one yamllint per slice, run concurrently. Each slice's output goes to its own file
|
||||
# and the files are printed in slice order afterwards: the log is the same text, in the same
|
||||
# file order, as one serial `yamllint "${files[@]}"`, and a finding is never interleaved with
|
||||
# another process's output (yamllint's standard format puts the FILE NAME on its own line above
|
||||
# that file's `line:col` findings, so interleaving would misattribute them). The stage's exit
|
||||
# code is the highest any slice returned (yamllint: 1 = error, 2 = warning under strict), so a
|
||||
# single failing slice fails the stage exactly as the serial call did. LINT_JOBS=1 is serial.
|
||||
lint_jobs="${LINT_JOBS:-$(n="$(nproc 2>/dev/null || echo 1)"; [ "$n" -gt 8 ] && n=8; echo "$n")}"
|
||||
case "$lint_jobs" in ''|*[!0-9]*|0) err "LINT_JOBS must be a positive integer, got '${lint_jobs}'"; exit 2 ;; esac
|
||||
[ "$lint_jobs" -gt "${#files[@]}" ] && lint_jobs="${#files[@]}"
|
||||
lint_tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$lint_tmp"' EXIT
|
||||
per=$(( (${#files[@]} + lint_jobs - 1) / lint_jobs ))
|
||||
pids=()
|
||||
for (( i = 0; i < lint_jobs; i++ )); do
|
||||
slice=("${files[@]:$(( i * per )):${per}}")
|
||||
[ "${#slice[@]}" -gt 0 ] || break
|
||||
( rc=0
|
||||
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${slice[@]}" \
|
||||
>"${lint_tmp}/${i}.out" 2>&1 || rc=$?
|
||||
echo "$rc" >"${lint_tmp}/${i}.rc" ) &
|
||||
pids+=("$!")
|
||||
done
|
||||
wait "${pids[@]}" || true
|
||||
lint_rc=0
|
||||
for (( i = 0; i < ${#pids[@]}; i++ )); do
|
||||
cat "${lint_tmp}/${i}.out"
|
||||
# A slice that left no rc file died before yamllint returned: that is a failure, not a pass.
|
||||
slice_rc="$(cat "${lint_tmp}/${i}.rc" 2>/dev/null || true)"
|
||||
case "$slice_rc" in ''|*[!0-9]*) slice_rc=1 ;; esac
|
||||
[ "$slice_rc" -gt "$lint_rc" ] && lint_rc="$slice_rc"
|
||||
done
|
||||
if [ "$lint_rc" -ne 0 ]; then
|
||||
err "yaml/lint failed (yamllint exit ${lint_rc}; ${#files[@]} files in ${#pids[@]} parallel slice(s))"
|
||||
exit "$lint_rc"
|
||||
fi
|
||||
ok "yaml/lint passed (${#files[@]} files, ${#pids[@]} parallel slice(s))"
|
||||
|
||||
# A workflow's `configurationFile:` must name a file that exists. Renovate's did not after #79
|
||||
# renamed renovate.json -> renovate.json5, and every scheduled run failed with ENOENT for 11 days
|
||||
|
||||
@@ -14,7 +14,7 @@ Pure standard library (matches the harness's no-pip-dependency posture); the che
|
||||
themselves shell out (render checks to catalog/render.py, whose PyYAML dependency lives
|
||||
there; L0 checks to git + PyYAML).
|
||||
|
||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] <test-dir> [<test-dir> ...]
|
||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] [--jobs N] <test-dir> [<test-dir> ...]
|
||||
Exit 0 iff every selected offline check passes.
|
||||
|
||||
The per-dir check script is whichever of `render-check.sh` / `check.sh` the directory
|
||||
@@ -43,13 +43,35 @@ Exit codes: 0 all selected checks passed / 1 at least one FAILED / 3 nothing fai
|
||||
scope-declared check was UNWIRED. conformance/run.sh propagates 3 verbatim; ci/l0.sh turns it
|
||||
into the gate's own rc 3. This is the ONE place the "unwired, not green" rule lives now,
|
||||
instead of once per hand-written stage.
|
||||
|
||||
PARALLELISM (--jobs, env CONF_OFFLINE_JOBS)
|
||||
-------------------------------------------
|
||||
The checks are independent processes, so they run in a bounded thread pool (one `bash` child
|
||||
per worker). Default: min(8, usable CPUs); `--jobs 1` is the old serial runner exactly. Each
|
||||
check's stdout+stderr is CAPTURED and only reported after every check has finished, in the
|
||||
ORDER THE DIRS WERE GIVEN, so the log, the JUnit and the step summary are byte-for-byte
|
||||
independent of which check finished first (only the trailing "[N.Ns]" wall-time on each
|
||||
status line varies). The 0/1/3 aggregation runs over the same ordered case list as before.
|
||||
|
||||
Parallel-safety is a property of the CHECKS, not of this runner: every check must keep its
|
||||
scratch state in a private mktemp/mkdtemp dir and must not write into the tree or a fixed
|
||||
path. The audit that established that for the current set is recorded in
|
||||
decisions/2026-11-08-parallel-l0-checks.md; a new check that needs shared state must say so.
|
||||
|
||||
Nested pools: the keycloak reconciler checks run their own mutation suites in a pool
|
||||
(mutations.py, env MUTATION_JOBS). When this runner is itself parallel it exports a SMALLER
|
||||
MUTATION_JOBS (unless the caller already set one), so the two suites share the CPUs the
|
||||
other jobs-2 checks are not using instead of each taking min(8, CPUs). See inner_jobs().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from xml.dom import minidom
|
||||
@@ -97,6 +119,46 @@ def collect_meta(test_yaml: Path, prefix: str):
|
||||
return name, reqs
|
||||
|
||||
|
||||
def usable_cpus() -> int:
|
||||
"""CPUs this process may run on (the cgroup/affinity view, not the host's count)."""
|
||||
try:
|
||||
return max(1, len(os.sched_getaffinity(0)))
|
||||
except (AttributeError, OSError):
|
||||
return max(1, os.cpu_count() or 1)
|
||||
|
||||
|
||||
def positive_int(text: str) -> int:
|
||||
n = int(text)
|
||||
if n < 1:
|
||||
raise argparse.ArgumentTypeError(f"must be >= 1, got {n}")
|
||||
return n
|
||||
|
||||
|
||||
def inner_jobs(cpus: int, outer: int) -> int:
|
||||
"""MUTATION_JOBS to hand the checks when this runner runs `outer` of them at once.
|
||||
|
||||
Two checks (the keycloak client-secret and passkey reconcilers) fan out internally; the
|
||||
other outer-2 workers are single processes. Give the two suites an equal share of the CPUs
|
||||
those others leave free, never fewer than 2 each (a 1-wide pool would make the 40-mutation
|
||||
passkey suite the critical path of the whole stage), never more than 8. Peak process count
|
||||
is then about (outer-2) + 2*inner, i.e. <= cpus on any machine with cpus >= outer + 2, and
|
||||
at worst ~1.25-1.5x cpus on a small runner — CPU-bound oversubscription of that size costs
|
||||
latency per process, not throughput.
|
||||
"""
|
||||
if outer <= 1:
|
||||
return min(8, cpus)
|
||||
return max(2, min(8, (cpus - outer + 2) // 2))
|
||||
|
||||
|
||||
def run_check(check: Path, env: dict) -> tuple[int, str, float]:
|
||||
t0 = time.monotonic()
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True,
|
||||
stdin=subprocess.DEVNULL, env=env,
|
||||
)
|
||||
return proc.returncode, (proc.stdout + proc.stderr).strip(), time.monotonic() - t0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--junit", type=Path, required=True)
|
||||
@@ -108,26 +170,54 @@ def main() -> int:
|
||||
"(makes this runner exit 3); repeatable")
|
||||
ap.add_argument("--summary-out", type=Path, default=None,
|
||||
help="write one Markdown line per case here, for the CI step summary")
|
||||
ap.add_argument("--jobs", type=positive_int, default=None,
|
||||
help="checks to run at once (default: env CONF_OFFLINE_JOBS, else "
|
||||
"min(8, usable CPUs)); 1 = serial")
|
||||
ap.add_argument("dirs", nargs="+", type=Path)
|
||||
args = ap.parse_args()
|
||||
|
||||
strict_scopes = set(args.unwired_scope)
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||
cpus = usable_cpus()
|
||||
if args.jobs is not None:
|
||||
jobs = args.jobs
|
||||
elif os.environ.get("CONF_OFFLINE_JOBS", "").strip():
|
||||
try:
|
||||
jobs = positive_int(os.environ["CONF_OFFLINE_JOBS"].strip())
|
||||
except (ValueError, argparse.ArgumentTypeError) as e:
|
||||
ap.error(f"CONF_OFFLINE_JOBS={os.environ['CONF_OFFLINE_JOBS']!r}: {e}")
|
||||
else:
|
||||
jobs = min(8, cpus)
|
||||
child_env = dict(os.environ)
|
||||
if not child_env.get("MUTATION_JOBS", "").strip():
|
||||
child_env["MUTATION_JOBS"] = str(inner_jobs(cpus, jobs))
|
||||
print(f"[offline] running {len(args.dirs)} check(s), jobs={jobs} "
|
||||
f"(cpus={cpus}, MUTATION_JOBS={child_env['MUTATION_JOBS']})", file=sys.stderr)
|
||||
|
||||
# Metadata first, in the given order; then the checks themselves through the pool. The
|
||||
# results are consumed in SUBMISSION order, so `cases` is identical to the serial runner's.
|
||||
planned = [] # (name, reqs, dir, scope, check-or-None)
|
||||
for d in args.dirs:
|
||||
check = next((d / s for s in CHECK_SCRIPTS if (d / s).exists()), None)
|
||||
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
||||
scope = scope_of(d / "chainsaw-test.yaml", args.scope_annotation)
|
||||
if check is None:
|
||||
cases.append((name, reqs, d, 3,
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||
scope))
|
||||
continue
|
||||
proc = subprocess.run(
|
||||
["bash", str(check)], capture_output=True, text=True
|
||||
)
|
||||
out = (proc.stdout + proc.stderr).strip()
|
||||
cases.append((name, reqs, d, proc.returncode, out, scope))
|
||||
planned.append((name, reqs, d, scope, check))
|
||||
|
||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||
elapsed = [] # wall seconds per case, same order (None: nothing ran)
|
||||
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
||||
futures = [pool.submit(run_check, check, child_env) if check is not None else None
|
||||
for (_n, _r, _d, _s, check) in planned]
|
||||
for (name, reqs, d, scope, check), fut in zip(planned, futures):
|
||||
if fut is None:
|
||||
cases.append((name, reqs, d, 3,
|
||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||
scope))
|
||||
elapsed.append(None)
|
||||
continue
|
||||
rc, out, secs = fut.result()
|
||||
cases.append((name, reqs, d, rc, out, scope))
|
||||
elapsed.append(secs)
|
||||
|
||||
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
||||
skipped = sum(1 for c in cases if c[3] == 3)
|
||||
@@ -145,7 +235,7 @@ def main() -> int:
|
||||
},
|
||||
)
|
||||
summary_lines = []
|
||||
for name, reqs, d, rc, out, scope in cases:
|
||||
for (name, reqs, d, rc, out, scope), secs in zip(cases, elapsed):
|
||||
tc = ET.SubElement(
|
||||
suite,
|
||||
"testcase",
|
||||
@@ -164,8 +254,9 @@ def main() -> int:
|
||||
fail.text = out
|
||||
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
||||
mark = f" scope={scope}" if scope else ""
|
||||
took = f" [{secs:.1f}s]" if secs is not None else ""
|
||||
print(f"[offline] {layer_of(d):3s} {status:4s} {name} "
|
||||
f"({','.join(reqs) or 'untagged'}){mark}",
|
||||
f"({','.join(reqs) or 'untagged'}){mark}{took}",
|
||||
file=sys.stderr)
|
||||
if rc not in (0, 3):
|
||||
print(out, file=sys.stderr)
|
||||
|
||||
@@ -25,15 +25,19 @@
|
||||
#
|
||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||
#
|
||||
# Runtime is ~20s (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster).
|
||||
# Runtime is ~11s serial (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||
# verdicts are independent of the pool size.
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -136,43 +140,99 @@ MUTATIONS: list[tuple[str, str, str, str]] = [
|
||||
]
|
||||
|
||||
|
||||
def mutation_jobs() -> int:
|
||||
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||
|
||||
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||
"""
|
||||
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||
if raw:
|
||||
try:
|
||||
n = int(raw)
|
||||
except ValueError:
|
||||
n = 0
|
||||
if n < 1:
|
||||
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||
return n
|
||||
try:
|
||||
cpus = len(os.sched_getaffinity(0))
|
||||
except (AttributeError, OSError):
|
||||
cpus = os.cpu_count() or 1
|
||||
return max(1, min(8, cpus))
|
||||
|
||||
|
||||
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(mutated_src)
|
||||
path = fh.name
|
||||
try:
|
||||
return subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=240,
|
||||
stdin=subprocess.DEVNULL)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||
return 2
|
||||
src = Path(sys.argv[1]).read_text()
|
||||
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
try:
|
||||
jobs = mutation_jobs()
|
||||
except ValueError as e:
|
||||
print("mutations.py: %s" % e, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||
for name, why, old, new in MUTATIONS:
|
||||
old, new = dedent4(old), dedent4(new)
|
||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||
# moved and this mutation silently stopped proving anything.
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
planned.append((name, why,
|
||||
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
"code moved and this mutation no longer proves anything. "
|
||||
"Re-anchor it against the current source." % (name, n))
|
||||
"Re-anchor it against the current source." % (name, n), None))
|
||||
continue
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(src.replace(old, new, 1))
|
||||
path = fh.name
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=240)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-42s %s" % (name, how))
|
||||
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||
|
||||
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||
try:
|
||||
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||
for (_n, _w, _a, mutated) in planned]
|
||||
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||
if anchor_problem is not None:
|
||||
problems.append(anchor_problem)
|
||||
continue
|
||||
proc = fut.result()
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-42s %s" % (name, how))
|
||||
finally:
|
||||
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||
# serially; queued mutations are cancelled rather than run for nothing.
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
print()
|
||||
if problems:
|
||||
|
||||
@@ -40,15 +40,19 @@
|
||||
#
|
||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||
#
|
||||
# Runtime is ~60s (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster).
|
||||
# Runtime is ~85s serial (one subprocess per mutation, each running the full selftest
|
||||
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||
# verdicts are independent of the pool size.
|
||||
# =============================================================================
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
# (name, the mistake being modelled, anchor, replacement)
|
||||
@@ -403,42 +407,98 @@ MUTATIONS: list[tuple[str, str, str, str]] = [
|
||||
]
|
||||
|
||||
|
||||
def mutation_jobs() -> int:
|
||||
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||
|
||||
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||
"""
|
||||
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||
if raw:
|
||||
try:
|
||||
n = int(raw)
|
||||
except ValueError:
|
||||
n = 0
|
||||
if n < 1:
|
||||
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||
return n
|
||||
try:
|
||||
cpus = len(os.sched_getaffinity(0))
|
||||
except (AttributeError, OSError):
|
||||
cpus = os.cpu_count() or 1
|
||||
return max(1, min(8, cpus))
|
||||
|
||||
|
||||
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(mutated_src)
|
||||
path = fh.name
|
||||
try:
|
||||
return subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
stdin=subprocess.DEVNULL)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||
return 2
|
||||
src = Path(sys.argv[1]).read_text()
|
||||
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
try:
|
||||
jobs = mutation_jobs()
|
||||
except ValueError as e:
|
||||
print("mutations.py: %s" % e, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||
for name, why, old, new in MUTATIONS:
|
||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||
# moved and this mutation silently stopped proving anything.
|
||||
n = src.count(old)
|
||||
if n != 1:
|
||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
planned.append((name, why,
|
||||
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||
"code moved and this mutation no longer proves anything. "
|
||||
"Re-anchor it against the current source." % (name, n))
|
||||
"Re-anchor it against the current source." % (name, n), None))
|
||||
continue
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||
fh.write(src.replace(old, new, 1))
|
||||
path = fh.name
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=300)
|
||||
finally:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-54s %s" % (name, how))
|
||||
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||
|
||||
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||
problems: list[str] = []
|
||||
killed = 0
|
||||
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||
try:
|
||||
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||
for (_n, _w, _a, mutated) in planned]
|
||||
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||
if anchor_problem is not None:
|
||||
problems.append(anchor_problem)
|
||||
continue
|
||||
proc = fut.result()
|
||||
if proc.returncode == 0:
|
||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||
"applied, so nothing in the suite actually checks it. The "
|
||||
"mistake it models: %s" % (name, why))
|
||||
continue
|
||||
killed += 1
|
||||
out = proc.stdout + proc.stderr
|
||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||
else "the selftest aborted (caught, less cleanly)")
|
||||
print(" killed %-54s %s" % (name, how))
|
||||
finally:
|
||||
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||
# serially; queued mutations are cancelled rather than run for nothing.
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
print()
|
||||
if problems:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: feat/devcontainer-velero-cli
|
||||
title: Bake the velero CLI into the dev toolbox, pinned to the prod server version
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
- `infra/devcontainer/versions.env`: new `VELERO_VERSION=1.18.2`, with a comment explaining that
|
||||
the CLI and server are version-coupled, so this pin follows the server and is not bumped on
|
||||
its own. It has **no** `# renovate:` marker on purpose. When a `velero` chart bump in
|
||||
`platform/velero/helmrelease.yaml` changes the appVersion, bump this pin in the same PR.
|
||||
- `infra/devcontainer/Containerfile`: `ARG VELERO_VERSION`; the binary comes from the
|
||||
vmware-tanzu GitHub release tarball and is installed in the same RUN as flux/helm. There is
|
||||
no checksum check, which matches the other tools in that file (none of them verify one).
|
||||
`velero=` is recorded in `/etc/dev-toolchain-versions`. The verify stage now checks that
|
||||
`velero` is on PATH, and `check velero v${VELERO_VERSION}` asserts the output of
|
||||
`velero version --client-only`.
|
||||
- `infra/devcontainer/build-image.sh`: `VELERO_VERSION` added to `BUILD_ARGS`. CI reads that
|
||||
list generically through `--print-build-args` (`.gitea/workflows/scripts/images.sh`), so
|
||||
**no sensitive path (`.gitea/**`, `ci/**`) was touched**.
|
||||
- `infra/devcontainer/README.md`: added a Backups row to the tool table.
|
||||
- `runbooks/backup-failure.md`: under "First commands", a read-only
|
||||
`velero backup describe --details` / `backup logs` example run through dev.sh.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Live server version, read-only: `kubectl --context k3s-prod -n velero get deploy velero`
|
||||
shows image `docker.io/velero/velero:v1.18.2`, and HelmRelease `velero` has
|
||||
lastAttemptedRevision `12.2.3`.
|
||||
- Local build: `IMAGE=localhost/home-k3s-dev:velero-wt ./build-image.sh` returned rc 0. The
|
||||
verify stage printed `ok velero` (on PATH), `ok velero v1.18.2` (pin check) and
|
||||
`TOOLBOX VERIFIED`. The image was built under a separate local tag and **not pushed**.
|
||||
- Smoke test with that image against prod (read-only): `velero version` reports Client v1.18.2
|
||||
and Server v1.18.2. `velero backup get` lists the daily backups, all `Completed` with 0 errors.
|
||||
- Recurring warning (reported only, not fixed): every daily backup since
|
||||
`velero-daily-20261031030050` has 1 warning. Backups up to 20261030 have 0. The warning text is:
|
||||
`gitea-actions: resource: /pods name: /act-runner-5944dfdfb8-69bvq message: /Skip pod volume
|
||||
data error: /pod is not in the expected status, ... phase=Failed: pod is not running`.
|
||||
It is the same pod in 20261031, 20261104 and 20261108. `gitea-actions` holds 11 `Failed`
|
||||
pods with reason `Evicted` from ReplicaSet `act-runner-5944dfdfb8`, ten of them started
|
||||
2026-10-30T08:26Z. The running runner is `act-runner-fc597f6b5-vsvkl`. So the warning comes from a
|
||||
leftover evicted (`Failed`) pod that Velero still tries to back up volumes for. Cleaning up the evicted pods is an
|
||||
operator action.
|
||||
- Gate: see the PR description for the toolbox `ci/gate.sh` rc and the skipped stages.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- The CI image build (`.gitea/workflows/images.yaml`) on this branch. It runs on push/PR and I
|
||||
have not seen its result while writing this.
|
||||
- An arm64 build. Only amd64 was built (`TARGETARCH` defaults to amd64).
|
||||
- Any mutating velero command (`backup create`, `restore`, `delete`). None were run.
|
||||
- Why only `-69bvq` of the 11 evicted act-runner pods shows up in the warning. A likely guess is
|
||||
that only that pod's volumes match the backup's pod-volume selection, but I did not check.
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
date: 2026-11-08
|
||||
branch: fix/keycloak-operator-crds
|
||||
title: "Keycloak 26.8.2 operator CrashLoop: install its two new CRDs, raise its memory"
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
**Incident.** PR #121 (Renovate) bumped `platform/keycloak/kustomization.yaml`'s three
|
||||
`keycloak-k8s-resources` URLs 26.0.6 -> 26.8.2. Upstream `kubernetes/` at 26.8.2 holds **four**
|
||||
CRDs, not two (GitHub contents API, `?ref=26.8.2`):
|
||||
|
||||
| file | at 26.0.6 | at 26.8.2 |
|
||||
|---|---|---|
|
||||
| `keycloaks.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||
| `keycloakrealmimports.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||
| `keycloakoidcclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||
| `keycloaksamlclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||
| `kubernetes.yml` | yes | yes (now carries RBAC for both new kinds) |
|
||||
|
||||
(`cluster-wide/` is a separate variant; upstream's own `kubernetes/kustomization.yml` lists exactly
|
||||
the five files above.) The 26.8.2 operator starts an informer for every controller it has and dies
|
||||
on the first missing kind: `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1
|
||||
... GET .../namespaces/keycloak/keycloaksamlclients ... Not Found`, exit 1, CrashLoopBackOff. Flux
|
||||
`platform` (`wait: true`, 5m) sat in health checks and every dependent Kustomization reported
|
||||
`dependency ... is not ready`. The old 26.0.6 ReplicaSet kept serving (RollingUpdate 25%/25%), so SSO
|
||||
stayed up.
|
||||
|
||||
**Fix.**
|
||||
- `platform/keycloak/kustomization.yaml`: added the two CRD URLs, each with the same
|
||||
`# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker (the repo's
|
||||
custom.regex manager matches all five — checked by running its regex over the file). Header
|
||||
comment updated to v26.8.2, plus a warning that Renovate cannot add a CRD a release introduces.
|
||||
- Same file: a `patches:` JSON6902 patch on Deployment `keycloak-operator`: memory limit
|
||||
450Mi -> **768Mi**, request 450Mi -> **512Mi**. 450Mi is upstream's default (unchanged at 26.8.2,
|
||||
so not something the bump changed). Evidence it is too small here: the 26.0.6 pod had **154
|
||||
restarts**, `lastState.terminated.reason: OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and
|
||||
`kubectl top` showed **421Mi / 450Mi**. 26.8.2 runs four controllers instead of two on Quarkus
|
||||
3.40.1.1 (was 3.15.1). No variable in the operator to disable watching unused kinds was used —
|
||||
installing the CRDs is upstream's supported shape.
|
||||
- `platform/keycloak/README.md` "Upgrading Keycloak": the diff-the-CRD-list step; "three URLs" ->
|
||||
"the release URLs". `renovate.json5`: same wording fix in a comment only.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Upstream file lists at 26.0.6 and 26.8.2 (GitHub API) and `diff` of `kubernetes.yml` between the
|
||||
two: the new kinds' ClusterRoles + RoleBindings ship in it, and the Deployment's resources are
|
||||
still 450Mi/450Mi.
|
||||
- Live, read-only (`kubectl --context k3s-prod`, 2026-11-08 ~05:57Z): only `keycloaks` and
|
||||
`keycloakrealmimports` CRDs installed; Deployment strategy RollingUpdate maxSurge/maxUnavailable
|
||||
25%; RS `keycloak-operator-58dff6c4` (26.8.2) 0/1 CrashLoopBackOff, RS `78dbb75f64` (26.0.6) 1/1;
|
||||
operator log is the informer error above; Flux `platform` `Healthy=Unknown` "Running health
|
||||
checks for revision main@sha1:e8d2666"; no NetworkPolicy and no CiliumNetworkPolicy in ns
|
||||
`keycloak`.
|
||||
- No sensitive path touched (`infra/gitea/approval-policy.json`), so this should merge on green.
|
||||
- Gate: see the PR body for the exact rc and stages of the run against the final commit.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **That the 26.8.2 operator becomes Ready after this merges.** Only the two missing kinds were
|
||||
observed failing; another start-up failure behind them is possible.
|
||||
- **That 768Mi is enough** — no 26.8.2 memory measurement exists yet; re-check `kubectl top` after
|
||||
a week.
|
||||
- **The Keycloak server upgrade 26.0.6 -> 26.8.2 itself.** Once the operator runs it will roll
|
||||
keycloak-0/1 and Keycloak migrates the DB schema. The CR's `spec.update.strategy` defaults to
|
||||
`RecreateOnImageChange` — expect a **full SSO outage** while both pods recreate, not a rolling
|
||||
one. Back up / confirm the latest CNPG backup first if that matters.
|
||||
- **The 26.8.2 Keycloak CRD's `spec.networkPolicy.enabled` defaults to `true`.** The operator will
|
||||
create a NetworkPolicy selecting the Keycloak pods (first policy in that namespace). With no
|
||||
`from` lists set it allows all sources on the HTTP/management ports, so Gateway, ServiceMonitor
|
||||
and the reconciler Jobs should still reach it — but this was read from the CRD text, not observed.
|
||||
- **The passkey reconciler's 26.0.6 Admin API assumptions against 26.8.2** (README "Upgrading
|
||||
Keycloak" list) — not re-checked here; PR #121 merged without it.
|
||||
- `catalog/keycloak.yaml`'s documentation-only image tag was not updated.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
date: 2026-11-08
|
||||
branch: feat/parallel-l0-checks
|
||||
title: L0 offline checks, mutation suites and yamllint run in bounded parallel pools (gate 184s -> 32s locally)
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
The gate's wall time was dominated by serial loops over independent subprocesses. Three of
|
||||
them now run in bounded pools; none of them changes a verdict, an exit code, or the order of
|
||||
anything printed.
|
||||
|
||||
- `conformance/lib/run_l1_offline.py` — the per-check loop (L0 and L1 offline checks) runs in
|
||||
a `ThreadPoolExecutor` of `bash <check>` children. `--jobs N` / env `CONF_OFFLINE_JOBS`,
|
||||
default `min(8, usable CPUs)` (`sched_getaffinity`, so a cgroup cpuset is respected);
|
||||
`--jobs 1` is the old serial runner. Results are consumed in SUBMISSION order, so `cases`,
|
||||
the JUnit, the step summary and the 0/1/3 aggregation are built from exactly the same
|
||||
ordered list as before. Additive output only: one `[offline] running N check(s), jobs=…`
|
||||
line, and a trailing `[N.Ns]` wall time on each per-check status line. Children get
|
||||
`stdin=/dev/null` (parallel children must not share the runner's stdin).
|
||||
- `conformance/tests/l0-static/keycloak-{passkey,client-secret}-reconciler/mutations.py` —
|
||||
pass 1 (serial) checks every anchor and builds each mutated source, exactly as before;
|
||||
pass 2 runs the `--selftest` children in a pool (env `MUTATION_JOBS`, default
|
||||
`min(8, CPUs)`, `1` = serial). Verdicts are consumed in `MUTATIONS` order. A timeout
|
||||
(`TimeoutExpired`) still propagates and fails the check; queued mutations are cancelled.
|
||||
An invalid `MUTATION_JOBS` exits 2 (fails the check, never skips it).
|
||||
- **Nested-pool budget.** When the outer runner is parallel it exports
|
||||
`MUTATION_JOBS = max(2, min(8, (cpus - jobs + 2) // 2))` unless the caller already set
|
||||
one. Reason: only those two checks fan out; the other `jobs-2` workers are single
|
||||
processes, so the two suites split the CPUs those leave free. Peak processes ≈
|
||||
`(jobs-2) + 2*inner` ≤ cpus whenever cpus ≥ jobs+2 (24 CPUs: 6 + 16 = 22), and ~1.25–1.5×
|
||||
cpus on a small runner (4 CPUs: 2 + 4 = 6). The floor of 2 is deliberate: a 1-wide inner
|
||||
pool would make the 40-mutation passkey suite (~85s serial) the critical path of the stage.
|
||||
- `ci/lint.sh` — yamllint runs over `LINT_JOBS` (default `min(8, nproc)`) CONTIGUOUS slices
|
||||
of the sorted file list concurrently; each slice's output is buffered to its own file and
|
||||
printed in slice order, so the text is the same as one serial call and a `file` header is
|
||||
never separated from its `line:col` findings. Stage rc = the highest slice rc; a slice that
|
||||
leaves no rc file counts as 1. Invalid `LINT_JOBS` exits 2.
|
||||
|
||||
### Parallel-safety audit (all 25 `l0-static/*/check.sh`, 3 `l1-rendering/*/render-check.sh`)
|
||||
|
||||
Grepped every check for fixed `/tmp` paths, `_artifacts`/`ARTIFACTS`/`COVERAGE_OUT`,
|
||||
`write_text`/`open(…,"w")`, `mktemp`/`mkdtemp`/`TemporaryDirectory`, sockets/ports, `git`
|
||||
writes and `run.sh` invocations, and read what each writer writes to:
|
||||
|
||||
- **Scratch state: all private.** Every writer uses its own `mktemp -d` / `mkdtemp` /
|
||||
`TemporaryDirectory` (placeholder-guard-coupling, spec-immutability-enforced,
|
||||
whole-repo-invariants-wired, auditability-proofs-producible, alert-rules-unit-tested and its
|
||||
`platform/monitoring/tests/run.sh`, internal-services-publish-no-public-dns,
|
||||
oauth2-proxy-{api-routes-per-app,cookie-names-distinct}, both keycloak reconcilers, the three
|
||||
L1 render checks). No fixed `/tmp` path anywhere.
|
||||
- **Git writes:** only spec-immutability-enforced, and only `git init/add/commit` inside its
|
||||
own `mktemp` repo. Everything else is `git ls-files`/`ls-tree`/`log`/`rev-parse` (read-only).
|
||||
- **Ports:** the reconciler selftests bind `127.0.0.1:0` (kernel-assigned); the passkey
|
||||
selftest also dials `127.0.0.1:1` as a deliberate refused connection. No fixed listener.
|
||||
- **Ansible** (dev-nodes-provisioned-as-code): `--syntax-check`/`--list-hosts` only,
|
||||
`retry_files_enabled = False`; its local tmp is per-process.
|
||||
- **The one shared writer:** auditability-proofs-producible runs
|
||||
`./run.sh --coverage-only --requirements …`, which REWRITES
|
||||
`conformance/_artifacts/coverage/coverage.{json,md}` (it inherits `CONF_ARTIFACTS`). That was
|
||||
already true serially. No other offline check reads or writes that directory (the only
|
||||
reader is `tests/meta/harness-selfcheck`, not an offline check), and the outer runner
|
||||
writes coverage BEFORE and the JUnit AFTER the pool, so the final state is the same as
|
||||
serial. Not serialised; left as is and recorded here. `lib/status.py` (run by that check)
|
||||
reads `_artifacts/report.junit.xml` as an advisory — written only after the pool, unchanged.
|
||||
- whole-repo-invariants-wired calls `run.sh --print-selection` (writes nothing) and the
|
||||
runner itself against synthetic dirs in its own `mktemp`.
|
||||
|
||||
Result: no check needed fixing or serialising. A new check that needs shared state must say
|
||||
so; the runner's docstring now states that contract.
|
||||
|
||||
## What I verified
|
||||
|
||||
All in the pinned toolbox (`dev.sh run`, `DEV_KUBECONFIG=/dev/null`, ENGINE shim adding a ro
|
||||
mount of the main checkout's `.git`; spec repo resolved as a sibling, spec-immutability
|
||||
`pass`), host 24 CPUs, every `dev.sh` call under `flock`.
|
||||
|
||||
- **Gate, before** (origin/main `1a33814`, detached scratch worktree + spec mount):
|
||||
`GATE_RC=3`, in-container wall **183.9s**; `l0` stage **168.3s**, `lint` **7.8s**.
|
||||
"Stages that did not run: gitea-settings" (plus the expected on-base
|
||||
touched-set-not-determinable note, since main has no branch commits).
|
||||
- **Gate, after** (this branch, commit `a775082`): `GATE_RC=3`, in-container wall **32.1s**;
|
||||
`l0` **21.1s**, `lint` **2.4s**, `l1` 2.3s. "Stages that did not run: gitea-settings" — the
|
||||
only not-run stage. **rc 3, not green.** Caveat: the branch run selected 23 L0 checks (the
|
||||
touched-requirement scoping dropped dev-nodes-provisioned-as-code and
|
||||
placeholder-guard-coupling), the main run 25.
|
||||
- **Same-tree L0 comparison** (all 25 l0 dirs, same tree, same container):
|
||||
`CONF_OFFLINE_JOBS=1 MUTATION_JOBS=1` **164.0s / 165.3s** vs default (jobs=8,
|
||||
MUTATION_JOBS=8) **19.2s / 18.7s**. JUnit `cmp`-identical, step summary `cmp`-identical,
|
||||
stderr identical except the timing suffixes and the JUnit path in the last line.
|
||||
- **Mutation suites, host:** passkey 83.2s → 10.5s (`MUTATION_JOBS=8`), client-secret
|
||||
11.2s → 2.0s; output of `MUTATION_JOBS=1` and `=8` byte-identical to the pre-change script's,
|
||||
rc 0 all round.
|
||||
- **Falsification (not committed):** in a scratch worktree at `a775082`, made the passkey
|
||||
mutation "the protected-subtree screen is never called" a no-op (replacement == anchor).
|
||||
Host: `MUTATION_JOBS=1` and `=8` both rc 1, output byte-identical (with a second, broken
|
||||
anchor also injected: both reported). Toolbox: runner rc 1 both serial and parallel,
|
||||
`L0 FAIL keycloak-passkey-reconciler … 1 of 40 mutation(s) not proven … SURVIVED`; full
|
||||
gate **rc 1**, `stage l0: FAIL`, `MERGE GATE: BLOCKED`. Reverted; scratch worktree removed.
|
||||
- **lint.sh, host scratch repo** (31 YAML files, two broken): `LINT_JOBS=1/4/8` all rc 1,
|
||||
stdout identical, findings named `y/f07.yaml` / `2:1 … key-duplicates` etc.; clean tree rc 0;
|
||||
`LINT_JOBS=x` rc 2.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **CI timings at the time of writing.** The PR's CI run is measured after push and reported
|
||||
on the PR / handoff message, not here. The CI runner's CPU count is UNVERIFIED — the new
|
||||
`[offline] running … jobs=… cpus=…` line in the CI log is what will show it. On a small
|
||||
runner the speedup will be smaller than locally.
|
||||
- **approval-policy** will be red on CI: `ci/**` and `conformance/**` are sensitive paths and
|
||||
need the operator's review. I did not and must not override it.
|
||||
- Flakiness under load: the selftests have no wall-clock assertions I found, and 4 toolbox
|
||||
runs + several host runs were all stable, but that is a handful of runs, not a soak.
|
||||
- The coverage files under `conformance/_artifacts/coverage/` end in the same state as before
|
||||
only because a single check rewrites them; I did not change that pre-existing quirk.
|
||||
- L2/L3 conformance: not run (no cluster in CI, as before).
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
date: 2026-11-08
|
||||
branch: chore/renovate-config-cleanup
|
||||
title: Renovate config — hand-migrate the docker key, track gitea/runner, keep HelmRelease range lower bounds
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
- **`renovate.json5`: the deprecated top-level `"docker": {"enabled": true}` is hand-migrated** to
|
||||
`packageRules[0] = {"matchCategories": ["docker"], "enabled": true}`, the exact shape Renovate's
|
||||
own migration produced on `origin/renovate/migrate-config` (PR #139). Its comment moved with it.
|
||||
This replaces #139, which was closed because its rewrite deleted every comment in the file. No
|
||||
other comment was removed or reflowed (diff is additions plus the moved block).
|
||||
- **`gitea/runner` added to the blast-radius rule** beside `gitea/act_runner`, with a comment:
|
||||
act_runner was renamed upstream to gitea/runner after 0.6.1; Renovate does not follow renames,
|
||||
so the actual image migration is a separate manual PR.
|
||||
- **Two new packageRules for flux/helm `>=A <B` ranges:** `rangeStrategy: "bump"`, plus
|
||||
`matchUpdateTypes: ["bump"] -> enabled: false`, so in-range releases (which Flux adopts on its
|
||||
own) do not become PRs. Cause of the bug, from the 44.167.0 source: rangeStrategy is unset
|
||||
(`auto`); the flux manager has no `getRangeStrategy`/`updateLockedDependency`, so
|
||||
`lib/modules/manager/index.ts getRangeStrategy` resolves `auto` to `replace`; helm versioning is
|
||||
the npm api verbatim, and npm `getNewValue` under `replace` turns `>=1.12.0 <1.13.0` into
|
||||
`<1.14.0` (what PRs #131, #130 and #115 did).
|
||||
- **`platform/headlamp/helmrelease.yaml`: `version: "<0.47.0"` -> `">=0.46.0 <0.47.0"`**, comment
|
||||
refreshed. The live chart is 0.46.0, so the set of installable charts is unchanged.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **Semantic diff of #139:** I parsed `renovate.json5` on `origin/renovate/migrate-config` and on its
|
||||
parent `5b484f6` with json5 (the copy inside the renovate image). With `docker` removed from the
|
||||
parent and `packageRules[0]` removed from the migrated file, the two are identical. So the only
|
||||
change was `{"matchCategories":["docker"],"enabled":true}`, inserted at packageRules[0].
|
||||
- **Validator, `ghcr.io/renovatebot/renovate:44.167.0`** (the tag that `.gitea/workflows/renovate.yaml`
|
||||
pins as `container.image`): `renovate-config-validator` gives `Config validated successfully`
|
||||
(rc 0), and so does `--strict` (rc 0), both as a file path and in repo mode. **`--strict` on
|
||||
origin/main's file gives rc 1** and prints the `docker` migration diff. Nothing is left to migrate.
|
||||
- **`getNewValue` called in the 44.167.0 image** (helm versioning), currentValue
|
||||
`>=1.12.0 <1.13.0`, newVersion 1.13.0: replace `<1.14.0`, widen `>=1.12.0 <1.14.0`,
|
||||
bump `>=1.13.0 <1.14.0`, pin/update-lockfile `<1.14.0`. On `<0.47.0`, bump returns the value
|
||||
unchanged, so it would never propose anything for that range. That is why the rule is scoped to
|
||||
the `>=A <B` shape.
|
||||
- **Local dry run** (`--platform=local`, `RENOVATE_DRY_RUN=lookup`, 44.167.0, live chart repos)
|
||||
against a fixture repo of four HelmReleases under `platform/`:
|
||||
- With this branch's config: `No config migration necessary`. longhorn `>=1.9.0 <1.10.0` ->
|
||||
`>=1.13.0 <1.14.0` (minor). kube-prometheus-stack `>=88.0.0 <89.0.0` -> `>=93.1.0 <94.0.0`
|
||||
(major). The in-range bump `>=88.6.5 <89.0.0` was dropped (`Filtered out 1 disabled update(s)`).
|
||||
headlamp `>=0.46.0 <0.47.0` and `<0.47.0`: no update.
|
||||
- Control run with origin/main's config: `Config migration necessary`, longhorn -> `<1.14.0`,
|
||||
kube-prometheus-stack -> `<94.0.0`. This reproduces the reported bug.
|
||||
- Live headlamp HelmRelease (read-only kubectl, k3s-prod): spec `<0.47.0`, history chartVersion
|
||||
`0.46.0`, Ready `True`.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **I have not seen a real scheduled Renovate run against this repo with this config.** The claim that
|
||||
Renovate stops reopening the migrate-config PR rests on the validator and on `No config migration
|
||||
necessary` in the local dry run, not on the Gitea bot. After merge, watch the next run, and if
|
||||
`renovate/migrate-config` is still on the remote, delete it.
|
||||
- **Under bump, the new lower bound is the release Renovate proposes**, e.g. `>=1.13.2 <1.14.0`, not
|
||||
`>=1.13.0`. Flux still installs the newest release in the window. Ranges with other shapes keep the
|
||||
default `replace`: `3.9.x`, `~`, ceiling-only, and exact pins.
|
||||
- **The open PRs #131, #115 and any others still carry unbounded `<X` values.** They were generated
|
||||
under the old strategy, and Renovate should rebase or regenerate them under bump on its next run.
|
||||
I have not checked that it does.
|
||||
- The gitea/runner rename itself (a new image reference and any config changes) is not done here.
|
||||
@@ -30,6 +30,9 @@ ARG CHECK_JSONSCHEMA_VERSION
|
||||
# Pinned to the Prometheus version prod actually runs — see versions.env for why that lockstep is
|
||||
# the point, and for the live-cluster command that establishes it.
|
||||
ARG PROMTOOL_VERSION
|
||||
# velero: operator CLI for the backup server. Pinned to the server version prod runs — the CLI
|
||||
# and server are version-coupled; see versions.env.
|
||||
ARG VELERO_VERSION
|
||||
ARG TARGETARCH=amd64
|
||||
|
||||
# Distro packages: libvirt client libs (harmless disk-image utilities), ansible, ssh,
|
||||
@@ -66,6 +69,10 @@ RUN set -eux; \
|
||||
curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl; chmod +x /usr/local/bin/kubectl; \
|
||||
curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" | tar -xz -C /tmp; \
|
||||
install -m0755 "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm; \
|
||||
VELERO_TARBALL="velero-v${VELERO_VERSION}-linux-${TARGETARCH}"; \
|
||||
curl -fsSL "https://github.com/vmware-tanzu/velero/releases/download/v${VELERO_VERSION}/${VELERO_TARBALL}.tar.gz" \
|
||||
| tar -xz -C /usr/local/bin --strip-components=1 "${VELERO_TARBALL}/velero"; \
|
||||
chmod +x /usr/local/bin/velero; \
|
||||
rm -rf /tmp/*
|
||||
|
||||
# Pinned conformance test toolchain (L0/L1 static gate + the L2/L3 Chainsaw runner).
|
||||
@@ -93,7 +100,7 @@ RUN { echo "opentofu=${OPENTOFU_VERSION}"; echo "age=${AGE_VERSION}"; echo "sops
|
||||
echo "kyverno=${KYVERNO_VERSION}"; echo "kyverno-json=${KYVERNO_JSON_VERSION}"; \
|
||||
echo "chainsaw=${CHAINSAW_VERSION}"; echo "yamllint=${YAMLLINT_VERSION}"; \
|
||||
echo "yq=${YQ_VERSION}"; echo "check-jsonschema=${CHECK_JSONSCHEMA_VERSION}"; \
|
||||
echo "promtool=${PROMTOOL_VERSION}"; \
|
||||
echo "promtool=${PROMTOOL_VERSION}"; echo "velero=${VELERO_VERSION}"; \
|
||||
} > /etc/dev-toolchain-versions
|
||||
|
||||
WORKDIR /work
|
||||
@@ -112,6 +119,7 @@ ARG KUBECTL_VERSION
|
||||
ARG CHAINSAW_VERSION
|
||||
ARG KYVERNO_VERSION
|
||||
ARG SOPS_VERSION
|
||||
ARG VELERO_VERSION
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for run --network none) ---"
|
||||
if getent hosts registry.fedoraproject.org >/dev/null 2>&1 || curl -sS --max-time 3 -o /dev/null https://1.1.1.1 2>/dev/null; then
|
||||
@@ -122,13 +130,13 @@ echo "--- baked versions ---"; cat /etc/dev-toolchain-versions
|
||||
echo "--- every tool is on PATH ---"
|
||||
for t in tofu age age-keygen sops flux kubectl helm kyverno kyverno-json chainsaw yq \
|
||||
yamllint check-jsonschema promtool ansible-playbook git make jq diff python3 \
|
||||
wg qrencode; do
|
||||
wg qrencode velero; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "toolbox is missing tools"; exit 1; }
|
||||
echo "--- version-sensitive tools report their pins ---"
|
||||
: "${PROMTOOL_VERSION:?}" "${KUBECTL_VERSION:?}" "${CHAINSAW_VERSION:?}" "${KYVERNO_VERSION:?}" "${SOPS_VERSION:?}"
|
||||
: "${PROMTOOL_VERSION:?}" "${KUBECTL_VERSION:?}" "${CHAINSAW_VERSION:?}" "${KYVERNO_VERSION:?}" "${SOPS_VERSION:?}" "${VELERO_VERSION:?}"
|
||||
check() { # name pin actual-output
|
||||
if printf "%s" "$3" | grep -qF "$2"; then printf " ok %-10s %s\n" "$1" "$2";
|
||||
else printf " DRIFT %-10s want %s, got: %s\n" "$1" "$2" "$(printf "%s" "$3" | head -1)"; drift=1; fi
|
||||
@@ -138,6 +146,7 @@ check kubectl "${KUBECTL_VERSION}" "$(kubectl version --client 2>&1)"
|
||||
check chainsaw "${CHAINSAW_VERSION}" "$(chainsaw version 2>&1)"
|
||||
check kyverno "${KYVERNO_VERSION}" "$(kyverno version 2>&1)"
|
||||
check sops "${SOPS_VERSION}" "$(sops --version --disable-version-check 2>&1 || sops --version 2>&1)"
|
||||
check velero "v${VELERO_VERSION}" "$(velero version --client-only 2>&1)"
|
||||
[ -z "${drift:-}" ] || { echo "a tool does not report its pinned version"; exit 1; }
|
||||
echo "--- python can import what the renderer and the gate need ---"
|
||||
python3 -c "import yaml; print(' PyYAML', yaml.__version__)"
|
||||
|
||||
@@ -14,6 +14,7 @@ baseline, and the *same* binaries CI uses.
|
||||
| Provisioning | OpenTofu (`bpg/proxmox` provider), `ansible-core`, `openssh-clients` |
|
||||
| GitOps / cluster | `flux`, `kubectl`, `helm` |
|
||||
| Secrets | `sops`, `age` + `age-keygen` |
|
||||
| Backups (`platform/velero`) | `velero` CLI — pinned to the **server** version prod runs (version-coupled; see `versions.env`) |
|
||||
| VPN (`platform/vpn-ingress`) | `wg` (wireguard-tools) — mint the server key, derive client public keys; `qrencode` — render a client profile as a phone-scannable QR |
|
||||
| **CI gate** | `yamllint`, `check-jsonschema`, `kyverno` CLI, `kyverno-json`, `chainsaw`, `yq`, `diffutils`, `make` |
|
||||
| Misc | `qemu-img`, `cloud-utils`/`genisoimage`, `libvirt-client` (disk-image utilities only — leftovers from the libvirt era, harmless) |
|
||||
|
||||
@@ -54,6 +54,7 @@ BUILD_ARGS=(
|
||||
"KYVERNO_JSON_VERSION=$KYVERNO_JSON_VERSION" "CHAINSAW_VERSION=$CHAINSAW_VERSION"
|
||||
"YAMLLINT_VERSION=$YAMLLINT_VERSION" "YQ_VERSION=$YQ_VERSION"
|
||||
"CHECK_JSONSCHEMA_VERSION=$CHECK_JSONSCHEMA_VERSION" "PROMTOOL_VERSION=$PROMTOOL_VERSION"
|
||||
"VELERO_VERSION=$VELERO_VERSION"
|
||||
)
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
|
||||
@@ -44,4 +44,19 @@ CHECK_JSONSCHEMA_VERSION=0.29.4
|
||||
# promtool ships INSIDE the prometheus release tarball (prometheus-<v>.linux-<arch>.tar.gz), so
|
||||
# the version string here is a Prometheus release, not a separate promtool release.
|
||||
PROMTOOL_VERSION=3.14.0
|
||||
# velero — the operator CLI for the Velero backup server (platform/velero): `velero backup
|
||||
# describe <name> --details`, `velero backup logs`, `velero backup create --from-schedule`.
|
||||
#
|
||||
# THIS PIN IS NOT FREE-FLOATING: the CLI and server are VERSION-COUPLED (the CLI talks to the
|
||||
# server's CRDs and its download-request protocol; upstream supports only a matching minor), so
|
||||
# it MUST equal the Velero server version actually running on prod.
|
||||
# Verified against the live cluster on 2026-11-07:
|
||||
# kubectl --context k3s-prod -n velero get deploy velero \
|
||||
# -o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
# -> docker.io/velero/velero:v1.18.2 (HelmRelease velero, chart velero-12.2.3)
|
||||
# Deliberately carries NO `# renovate:` marker: it must move WITH the server, i.e. when a
|
||||
# vmware-tanzu `velero` chart bump (platform/velero/helmrelease.yaml, range >=12 <13) changes the
|
||||
# chart's appVersion, bump this in the same PR. Never bump it on its own. The verify stage
|
||||
# asserts `velero version --client-only` reports exactly this version.
|
||||
VELERO_VERSION=1.18.2
|
||||
# Proxmox provider (bpg/proxmox) is pinned in the tofu module (infra/tofu/versions.tf), not here.
|
||||
@@ -40,7 +40,11 @@ spec:
|
||||
# values (-session-ttl + the -oidc-* args from the externalSecret; useCookie stays default-false
|
||||
# so `-oidc-use-cookie` is NOT emitted) is supported by the v0.41.0 backend. Renovate maintains
|
||||
# within 0.41.x; the appVersion continues to pin the Headlamp image.
|
||||
version: "<0.47.0"
|
||||
# 2026-11-08: NOW 0.46.x. PR #130 (Renovate, default rangeStrategy `replace`) rewrote
|
||||
# `>=0.41.0 <0.42.0` to the unbounded-below `<0.47.0`; the lower bound is restored here at the
|
||||
# chart that was live when it merged (0.46.0). renovate.json5 now bumps `>=A <B` ranges as a window,
|
||||
# so the next widen PR proposes `>=0.47.x <0.48.0` rather than `<0.48.0`.
|
||||
version: ">=0.46.0 <0.47.0"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: headlamp
|
||||
|
||||
@@ -367,9 +367,17 @@ is involved — so step 2 failing with no log entry is the expected signature of
|
||||
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
||||
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
||||
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
|
||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the release URLs in
|
||||
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
||||
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
||||
|
||||
**Renovate bumps the URLs that exist; it cannot add a CRD a release introduced.** Before merging a
|
||||
bump, diff `kubernetes/` (and its `kustomization.yml`) at the new tag against the URL list. 26.0.6
|
||||
shipped 2 CRDs, 26.8.2 ships 4 (`keycloakoidcclients`, `keycloaksamlclients` added); PR #121 bumped
|
||||
only the existing URLs and the 26.8.2 operator CrashLooped on the missing `keycloaksamlclients`
|
||||
informer, stalling Flux `platform` and every tier below it (2026-11-08,
|
||||
`decisions/2026-11-08-keycloak-operator-crds.md`). The operator's memory is also patched in
|
||||
`kustomization.yaml` (upstream's 450Mi OOMKilled 26.0.6 here); keep the patch across bumps.
|
||||
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
||||
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
||||
|
||||
|
||||
@@ -20,19 +20,33 @@ kind: Kustomization
|
||||
namespace: keycloak
|
||||
resources:
|
||||
- namespace.yaml
|
||||
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
||||
# --- Keycloak Operator v26.8.2 (pinned upstream release manifests) ---
|
||||
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
||||
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
||||
# is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
|
||||
# is what runs. These URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all of them bump in ONE
|
||||
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
||||
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
||||
#
|
||||
# THE URL LIST MUST MATCH UPSTREAM'S kubernetes/kustomization.yml AT THE NEW TAG — Renovate only
|
||||
# rewrites the version in the URLs that are here; it cannot add a CRD the new release introduced.
|
||||
# 26.0.6 shipped 2 CRDs; 26.8.2 ships 4 (+ keycloakoidcclients, keycloaksamlclients, both
|
||||
# v2alpha1). PR #121 bumped only the existing three URLs, and the 26.8.2 operator — which starts an
|
||||
# informer for EVERY kind it controls, used or not — CrashLooped on
|
||||
# `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1 ... Not Found`,
|
||||
# stalling Flux `platform` and everything downstream (2026-11-08). On every bump, diff
|
||||
# https://github.com/keycloak/keycloak-k8s-resources/tree/<tag>/kubernetes
|
||||
# against this list before merging.
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakoidcclients.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaksamlclients.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
||||
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
||||
- keycloak-admin-credentials.sops.yaml
|
||||
# --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
|
||||
@@ -51,3 +65,23 @@ resources:
|
||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||
- grafana-dashboard.yaml
|
||||
# --- Operator memory: upstream ships requests=limits=450Mi, which this cluster outgrew ON 26.0.6 ---
|
||||
# Measured on prod 2026-11-08: the 26.0.6 operator pod had 154 restarts in 64d, lastState
|
||||
# `OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and sat at 421Mi of its 450Mi (94%) in `kubectl top`.
|
||||
# 26.8.2 keeps the same 450Mi default but runs FOUR controllers/informers instead of two (adds
|
||||
# keycloakoidcclient + keycloaksamlclient) on a newer Quarkus (3.15.1 -> 3.40.1.1), so it would
|
||||
# plausibly start above a ceiling 26.0.6 already crossed. Limit 768Mi = ~1.8x the observed 421Mi
|
||||
# working set; request 512Mi = the observed working set plus headroom, so the scheduler reserves what
|
||||
# it actually uses. Revisit with `kubectl top` once 26.8.2 has run for a week. Patched here (not in
|
||||
# the remote kubernetes.yml) so a Renovate bump keeps it.
|
||||
patches:
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: keycloak-operator
|
||||
patch: |-
|
||||
- op: replace
|
||||
path: /spec/template/spec/containers/0/resources/limits/memory
|
||||
value: 768Mi
|
||||
- op: replace
|
||||
path: /spec/template/spec/containers/0/resources/requests/memory
|
||||
value: 512Mi
|
||||
+64
-11
@@ -37,6 +37,8 @@
|
||||
// renovate-config-validator /tmp/renovate.json5
|
||||
// (Not `--strict`: that also fails on a pending migration of the pre-existing top-level
|
||||
// `"docker": {"enabled": true}` block at the bottom, which is a warning, not an error.)
|
||||
// RESOLVED 2026-11-08: that block is now hand-migrated to the first packageRule (`matchCategories:
|
||||
// ["docker"]`), the exact shape Renovate's own migration produced, so no migration is pending.
|
||||
//
|
||||
// `regex` vs `custom.regex` in enabledManagers, RESOLVED FROM SOURCE (Renovate 39.264.0 and
|
||||
// later): both work. lib/config/migrations/custom/enabled-managers-migration.ts rewrites
|
||||
@@ -196,6 +198,22 @@
|
||||
}
|
||||
],
|
||||
"packageRules": [
|
||||
// The `docker` manager (Dockerfiles, docker-compose) and the `kubernetes` manager above were
|
||||
// BOTH previously `enabled: false`, solely so Renovate would not fight Flux image-automation
|
||||
// over a running tag. That controller was never enabled and never will be here (see _comment),
|
||||
// so the only thing those two flags were achieving was leaving every plain `image:` tag in the
|
||||
// repo unmanaged. `docker` is left out of enabledManagers because this repo contains no
|
||||
// Dockerfile that ships to the cluster; `kubernetes` is now ON and is the one that matters.
|
||||
// MOVED HERE 2026-11-08 from a top-level `"docker": {"enabled": true}` key, which Renovate
|
||||
// deprecated (the top-level language objects became `matchCategories` rules). Hand-migrated to
|
||||
// the exact output of Renovate's own config migration (branch renovate/migrate-config, PR #139:
|
||||
// a json5 semantic diff of that file against its parent shows THIS rule, inserted at
|
||||
// packageRules[0], as the ONLY change). #139 itself was closed unmerged because its rewrite
|
||||
// dropped every comment in this file. With nothing left to migrate, Renovate stops reopening it.
|
||||
{
|
||||
"matchCategories": ["docker"],
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"description": "Auto-merge the allowed low-risk classes on green CI \u2014 no human authoring (R-UPD-02). The Gitea merge gate (make -C ci gate) is the required check.",
|
||||
"matchUpdateTypes": [
|
||||
@@ -252,6 +270,11 @@
|
||||
// Keycloak's (the IdP) included, now that imageName is tracked.
|
||||
"gitea/gitea",
|
||||
"gitea/act_runner",
|
||||
// ADDED 2026-11-08. act_runner was RENAMED upstream to gitea/runner after 0.6.1
|
||||
// (https://gitea.com/gitea/runner/releases). Renovate does not follow a rename — it will
|
||||
// never propose gitea/act_runner -> gitea/runner — so that migration is a manual PR. Listed
|
||||
// now so the new name is blast-radius from its first reference, not after an auto-merge.
|
||||
"gitea/runner",
|
||||
"ghcr.io/cloudnative-pg/postgresql"
|
||||
],
|
||||
"automerge": false,
|
||||
@@ -264,7 +287,7 @@
|
||||
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
||||
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
||||
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
||||
// 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||
// 26.0.6). The release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
||||
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
||||
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
||||
@@ -368,6 +391,45 @@
|
||||
"automerge": false,
|
||||
"addLabels": ["needs-review", "range-widen"]
|
||||
},
|
||||
// ADDED 2026-11-08 — KEEP THE LOWER BOUND WHEN A MINOR-LOCKED RANGE MOVES. MEASURED PROBLEM:
|
||||
// PRs #131 (longhorn), #130 (headlamp) and #115 (kube-prometheus-stack) turned
|
||||
// `">=1.12.0 <1.13.0"` into `"<1.14.0"` — an UNBOUNDED-below range that also admits every
|
||||
// older chart (and #130 merged: platform/headlamp/helmrelease.yaml carried `<0.47.0`). WHY,
|
||||
// from the 44.167.0 source this file is validated against: no rangeStrategy is set, so it is
|
||||
// `auto`; the flux manager has no getRangeStrategy and no updateLockedDependency, so
|
||||
// lib/modules/manager/index.ts getRangeStrategy resolves `auto` -> `replace`. The helm
|
||||
// datasource's defaultVersioning is `helm`, which is the npm versioning api verbatim
|
||||
// (lib/modules/versioning/helm/index.ts: `api = { ...npm }`), and npm getNewValue under
|
||||
// `replace` rewrites `>=A <B` to just `<C`. Calling that getNewValue in the 44.167.0 image for
|
||||
// currentValue `>=1.12.0 <1.13.0`, newVersion 1.13.0:
|
||||
// replace -> `<1.14.0` widen -> `>=1.12.0 <1.14.0` bump -> `>=1.13.0 <1.14.0`
|
||||
// Only `bump` keeps a one-minor window. (widen keeps the bound but grows the window by a minor
|
||||
// on every PR — the opposite of minor-locking.) Under bump the new lower bound is the release
|
||||
// proposed (e.g. 1.13.2 -> `>=1.13.2 <1.14.0`); Flux still picks the newest that satisfies it.
|
||||
// Scoped to the bounded `>=A <B` shape only: on a ceiling-only range like `<0.47.0` bump
|
||||
// returns the value unchanged, so Renovate would silently propose NOTHING for it, ever.
|
||||
{
|
||||
"description": "ADDED 2026-11-08. HelmRelease `version: \">=A <B\"` ranges move as a whole window (rangeStrategy bump), so a range-widen PR yields `>=1.13.0 <1.14.0`, not `<1.14.0`. See the comment above for the measured cause.",
|
||||
"matchManagers": ["flux"],
|
||||
"matchDatasources": ["helm"],
|
||||
"matchCurrentValue": "/^>=\\S+ <\\S+$/",
|
||||
"rangeStrategy": "bump"
|
||||
},
|
||||
// ...AND DO NOT LET bump ADD NOISE. Under bump Renovate also raises the lower bound for a
|
||||
// release ALREADY INSIDE the range (lib/workers/repository/process/lookup/generate.ts sets
|
||||
// isBump; e.g. 1.12.5 against `>=1.12.0 <1.13.0` -> `>=1.12.5 <1.13.0`), which `replace` never
|
||||
// proposed. Flux already installs that release with no PR, and the range-widen rule above
|
||||
// would hold every one for review. util/package-rules/update-types.ts matches isBump as
|
||||
// updateType `bump`, so exactly those updates are switched off here; out-of-range moves are
|
||||
// ordinary minor/major updates and are unaffected.
|
||||
{
|
||||
"description": "ADDED 2026-11-08. No PR for a chart release the HelmRelease range already admits — Flux adopts it on its own.",
|
||||
"matchManagers": ["flux"],
|
||||
"matchDatasources": ["helm"],
|
||||
"matchCurrentValue": "/^>=\\S+ <\\S+$/",
|
||||
"matchUpdateTypes": ["bump"],
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "ADDED 2026-10-30. velero-plugin-for-aws is pinned to a RELEASE CANDIDATE (v1.14.3-rc.2) and Renovate proposed NOTHING for it, although v1.14.3 and v1.14.4 are stable on Docker Hub. Docker versioning treats everything after the first `-` as a compatibility SUFFIX, so only other `*-rc.2` tags qualify and the RC is a fossil forever. Semver versioning reads it as a prerelease, and a prerelease upgrades to the stable release.",
|
||||
"matchPackageNames": ["velero/velero-plugin-for-aws"],
|
||||
@@ -439,14 +501,5 @@
|
||||
"conformance/tests/**",
|
||||
"platform/image-automation/per-service/**",
|
||||
"platform/kyverno/tests/**"
|
||||
],
|
||||
// The `docker` manager (Dockerfiles, docker-compose) and the `kubernetes` manager above were
|
||||
// BOTH previously `enabled: false`, solely so Renovate would not fight Flux image-automation
|
||||
// over a running tag. That controller was never enabled and never will be here (see _comment),
|
||||
// so the only thing those two flags were achieving was leaving every plain `image:` tag in the
|
||||
// repo unmanaged. `docker` is left out of enabledManagers because this repo contains no
|
||||
// Dockerfile that ships to the cluster; `kubernetes` is now ON and is the one that matters.
|
||||
"docker": {
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -99,6 +99,16 @@ infra/devcontainer/dev.sh run bash -c \
|
||||
-o custom-columns=NAME:.metadata.name,BACKUP:.metadata.labels.velero\\.io/backup-name,PHASE:.status.phase,BYTES:.status.progress.bytesDone'
|
||||
```
|
||||
|
||||
The `velero` CLI is in the toolbox (pinned to the server version) and gives the per-item errors and
|
||||
warnings the CR only counts. Read-only; `velero backup create/delete` and `velero restore` are
|
||||
operator actions:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'velero --kubecontext k3s-prod -n velero backup describe <name> --details; \
|
||||
velero --kubecontext k3s-prod -n velero backup logs <name> | grep -iE "level=(error|warn)" | tail -25'
|
||||
```
|
||||
|
||||
The plugin pin (cause 1 depends entirely on this):
|
||||
|
||||
```
|
||||
|
||||
Reference in new issue
Block a user