Home-Network/home-network-k3s-impl
2
0 Fork 0
Code Issues 3 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

refactor(apps): split monolithic apps Kustomization into dependency waves

Browse Source 
Replaces the single `apps` Flux Kustomization with dependency-ordered waves
(apps-wave-0..2) so a crashlooping workload isolates to its wave + downstream
instead of stalling the entire app tier. Parallel within a wave, ordered across.

- catalog/render.py: compute waves by longest dependency-path depth over the
  ServiceDefinition `dependencies:` edges (a `<svc>-db` is a leaf, so its
  consumer lands strictly higher — the "DB below consumer" rule falls out of
  longest-path, no special case). `--emit-waves` generates the per-wave
  kustomizations + Flux CRs; `--wave-layout` places renders into the right dir.
  wave 0: pihole gitea jackett plex qbittorrent nextcloud-db semaphore-db
  wave 1: sonarr radarr nextcloud semaphore gitea-actions
  wave 2: seerr
- apps/*.yaml -> apps/wave-{0,1,2}/ (byte-identical moves; SOPS files intact).
- clusters/base: apps.yaml removed; apps-wave-{0,1,2}.yaml added (dependsOn
  chain: platform-config -> wave-0 -> wave-1 -> wave-2, prune+wait each).
- clusters/dev: one SOPS decryption patch per wave (load-bearing).
- DESIGN-waves.md: wave table, topology, cutover runbook.

Cutover: preceded by disabling prune on `apps` (1bfb0db) so removing that
object here does not GC its resources; the wave Kustomizations SSA-adopt the
byte-identical manifests in place (no restart, no PVC churn).

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed 2026-08-26 12:19:00 -05:00
1 parent 1bfb0db6f1
commit 36a1a4e1f9
34 files changed
+793 -93
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

catalog/render.py
+349 -6
View File
@@ -935,6 +935,239 @@ def render_file(path: Path, image_automation: bool = True, catalog_dir: Path | N
return emit(render(sd, image_automation=image_automation, inbound=inbound))
# =============================================================================
# Wave partitioning — dependency-ordered app-tier delivery (WS-APPS wave delivery)
# =============================================================================
# A single monolithic `apps` Flux Kustomization health-gates (wait:true) the ENTIRE app
# set, so one crashlooping workload blocks ALL apps and every new revision. Instead we split
# the tier into dependency-ordered WAVES: each wave is its own Flux Kustomization
# `apps-wave-<N>` that dependsOn wave N-1 (wave 0 dependsOn platform-config), so a failure in
# a later wave can no longer stall an earlier one, while everything WITHIN a wave still
# reconciles in parallel. This module computes the wave assignment from the declared catalog
# dependency graph and emits the per-wave kustomize + Flux Kustomization CRs. It changes only
# WHERE a rendered manifest is placed — never its bytes (the L1 goldens stay green).
CO_DB_SUFFIX = "-db" # a co-located CNPG database for `<svc>` is the node `<svc>-db`.
# Non-catalog app-tier units (they carry NO ServiceDefinition, so compute_waves can't see
# them). Each is anchored to a catalog service; `above=True` forces it into the wave STRICTLY
# ABOVE its anchor because it is a hard runtime prerequisite it cannot start without (e.g. the
# Gitea Actions act_runner registers against + clones from Gitea, so Gitea must be healthy
# first); `above=False` would co-locate it in the anchor's own wave.
EXTRA_UNITS: dict = {
"gitea-actions": {
"anchor": "gitea",
"above": True,
"files": [
"gitea-actions.yaml",
"gitea-actions-runner-secret.sops.yaml",
],
},
}
def compute_waves(sds: list) -> dict:
"""Longest-dependency-path wave per node over the catalog dependency DAG.
Nodes = every catalog service ∪ every dependency target. A dependency target that has NO
ServiceDefinition (a co-located `<svc>-db` CNPG database) is a leaf. wave(n) = 0 if n
depends on nothing, else 1 + max(wave(dep)) over its declared dependencies. Because a
co-located DB is a leaf and its consumer declares a dependency ON it, the consumer always
lands in a strictly higher wave — the "DB strictly below its consumer" guarantee falls
straight out of the longest-path rule (no special case). Raises ValueError on a cycle.
"""
edges: dict = {} # node -> set(targets it depends on)
nodes: set = set()
for sd in sds:
if not isinstance(sd, dict) or "name" not in sd:
continue # tolerate non-ServiceDefinition siblings (e.g. render.py itself)
s = apply_defaults(sd)
name = s["name"]
nodes.add(name)
edges.setdefault(name, set())
for dep in s["dependencies"]:
edges[name].add(dep["to"])
nodes.add(dep["to"])
edges.setdefault(dep["to"], set())
wave: dict = {}
visiting: set = set()
def depth(n: str) -> int:
if n in wave:
return wave[n]
if n in visiting:
raise ValueError(f"dependency cycle through {n!r}")
visiting.add(n)
d = 0
for t in sorted(edges.get(n, ())):
d = max(d, depth(t) + 1)
visiting.discard(n)
wave[n] = d
return d
for n in sorted(nodes):
depth(n)
return wave
def _catalog_by_name(sds: list) -> dict:
out = {}
for sd in sds:
if isinstance(sd, dict) and "name" in sd:
s = apply_defaults(sd)
out[s["name"]] = s
return out
def service_wave(sds: list, name: str) -> int:
"""The wave a single service's rendered manifest belongs in (for wave-aware `-o`)."""
return compute_waves(sds).get(name, 0)
def plan_apps_waves(sds: list, present_files: set) -> dict:
"""Map the ONBOARDED app-tier file set into waves -> sorted resource basenames.
`present_files` is the set of manifest basenames currently committed under apps/ (flat or
already in wave-<N>/ subdirs), e.g. {"pihole.yaml", "nextcloud-db.yaml",
"nextcloud-db-app.sops.yaml", ...}. A catalog service is ONBOARDED iff `<svc>.yaml` is
present (the catalog also holds not-yet-onboarded SDs like keycloak/vm-demo, which must
NOT be placed). Each onboarded unit's sidecar files inherit the correct wave by naming
convention:
- `<svc>.yaml` -> wave(svc)
- `<svc>-bootstrap-secret.sops.yaml` -> wave(svc) (consumed by svc's env, same apply)
- `<svc>-db.yaml` -> wave(svc-db) (a strictly-lower wave; CNPG DB)
- `<svc>-db-app.sops.yaml` -> wave(svc-db) (CNPG initdb owner secret)
Non-catalog units (EXTRA_UNITS, e.g. gitea-actions) place by their declared anchor+offset.
Only files that actually exist in `present_files` are placed, so the plan is robust to a
service having no bootstrap secret or no DB. Deterministic (sorted).
"""
waves = compute_waves(sds)
by_name = _catalog_by_name(sds)
plan: dict = {}
def add(w: int, f: str) -> None:
if f in present_files:
plan.setdefault(w, set()).add(f)
onboarded = {n for n in by_name if f"{n}.yaml" in present_files}
for name in sorted(onboarded):
w = waves[name]
add(w, f"{name}.yaml")
add(w, f"{name}-bootstrap-secret.sops.yaml")
db = f"{name}{CO_DB_SUFFIX}"
if any(dep["to"] == db for dep in by_name[name]["dependencies"]):
wdb = waves[db]
add(wdb, f"{db}.yaml")
add(wdb, f"{db}-app.sops.yaml")
for unit, cfg in EXTRA_UNITS.items():
anchor = cfg["anchor"]
if anchor not in onboarded:
continue
w = waves[anchor] + (1 if cfg.get("above") else 0)
for f in cfg["files"]:
add(w, f)
return {w: sorted(fs) for w, fs in plan.items()}
def wave_kustomization_text(wave: int, files: list) -> str:
"""The kustomize Kustomization for one wave directory (apps/wave-<N>/kustomization.yaml)."""
dep = "platform-config" if wave == 0 else f"apps-wave-{wave - 1}"
header = (
f"# Wave {wave} of the app tier — GENERATED by `catalog/render.py --emit-waves`.\n"
f"# Reconciled by the Flux Kustomization `apps-wave-{wave}` "
f"(clusters/base/apps-wave-{wave}.yaml), which dependsOn `{dep}` and health-gates\n"
f"# (wait:true) ONLY this wave — a crashloop here cannot stall an earlier wave.\n"
f"# DO NOT hand-edit: re-run `render.py --emit-waves` after onboarding a service."
)
body = {
"apiVersion": "kustomize.config.k8s.io/v1beta1",
"kind": "Kustomization",
"resources": list(files),
}
dumped = yaml.dump(body, sort_keys=False, default_flow_style=False, width=4096)
return f"---\n{header}\n{dumped}"
def flux_wave_kustomization_text(wave: int) -> str:
"""The Flux Kustomization CR for one wave (clusters/base/apps-wave-<N>.yaml).
dependsOn wave N-1 (wave 0 dependsOn platform-config); prune+wait match the retired
monolith. SOPS decryption is added by the environment overlay patch, IDENTICALLY to the
former single `apps` Kustomization (clusters/dev patches each apps-wave-<N> by name) — the
base stays env-agnostic and references the out-of-band `sops-age` Secret by name only.
"""
dep = "platform-config" if wave == 0 else f"apps-wave-{wave - 1}"
header = (
f"# Flux Kustomization: app-tier WAVE {wave}. Reconciles ./apps/wave-{wave} AFTER "
f"`{dep}` is healthy.\n"
f"# GENERATED by `catalog/render.py --emit-waves`. Waves isolate a crashloop to its "
f"own wave + downstream:\n"
f"# wait:true health-gates ONLY this wave's resources, so a later wave can no longer "
f"stall an earlier one.\n"
f"# SOPS decryption is layered on per-environment (clusters/dev patches this "
f"Kustomization by name), identical to the retired `apps` Kustomization."
)
cr = {
"apiVersion": "kustomize.toolkit.fluxcd.io/v1",
"kind": "Kustomization",
"metadata": {"name": f"apps-wave-{wave}", "namespace": "flux-system"},
"spec": {
"interval": "10m",
"retryInterval": "2m",
"timeout": "5m",
"prune": True,
"wait": True,
"dependsOn": [{"name": dep}],
"sourceRef": {"kind": "GitRepository", "name": "flux-system"},
"path": f"./apps/wave-{wave}",
"postBuild": {
"substituteFrom": [{"kind": "ConfigMap", "name": "cluster-vars"}]
},
},
}
dumped = yaml.dump(cr, sort_keys=False, default_flow_style=False, width=4096)
return f"---\n{header}\n{dumped}"
def _discover_app_files(apps_dir: Path) -> set:
"""Basenames of every committed app-tier manifest under apps/ (flat OR in wave-<N>/).
Excludes the generated per-wave kustomization.yaml and any README. Deterministic.
"""
present = set()
for p in apps_dir.rglob("*.yaml"):
if p.name == "kustomization.yaml":
continue
present.add(p.name)
return present
def emit_waves(catalog_dir: Path, apps_dir: Path, clusters_dir: Path) -> dict:
"""Write apps/wave-<N>/kustomization.yaml + clusters/base/apps-wave-<N>.yaml.
Computes the wave plan from the catalog DAG and the app files present under apps/, then
(re)generates the per-wave kustomize files and the Flux Kustomization CRs. Idempotent:
it never moves or rewrites a service manifest (only its placement changes, done in git),
so the rendered bytes — and the L1 goldens — are untouched. Returns the wave plan.
"""
sds = _load_catalog(catalog_dir)
present = _discover_app_files(apps_dir)
plan = plan_apps_waves(sds, present)
for wave in sorted(plan):
wdir = apps_dir / f"wave-{wave}"
wdir.mkdir(parents=True, exist_ok=True)
(wdir / "kustomization.yaml").write_text(
wave_kustomization_text(wave, plan[wave])
)
(clusters_dir / f"apps-wave-{wave}.yaml").write_text(
flux_wave_kustomization_text(wave)
)
return plan
# =============================================================================
# Self-test (deterministic, no cluster) — `render.py --selftest`
# =============================================================================
@@ -1187,6 +1420,74 @@ def _selftest() -> int:
"an SD without podSecurityContext must not gain a pod securityContext",
)
# 7) wave partitioning: longest-path over the dependency DAG, CNPG-DB strictly below.
wsds = [
{"name": "a", "source": {"image": "x:1"}}, # leaf
{"name": "b", "source": {"image": "x:1"}}, # leaf
{"name": "c", "source": {"image": "x:1"},
"dependencies": [{"to": "a", "port": 1}, {"to": "b", "port": 1}]}, # wave 1
{"name": "d", "source": {"image": "x:1"},
"dependencies": [{"to": "c", "port": 1}]}, # wave 2
{"name": "svc", "source": {"image": "x:1"},
"dependencies": [{"to": "svc-db", "port": 5432}]}, # DB consumer
]
w = compute_waves(wsds)
check(w["a"] == 0 and w["b"] == 0, "leaf services must be wave 0")
check(w["c"] == 1, "a service depending on wave-0 nodes must be wave 1")
check(w["d"] == 2, "longest-path depth must be transitive (d -> c -> {a,b})")
check(
w["svc-db"] == 0 and w["svc"] == 1 and w["svc-db"] < w["svc"],
"a co-located <svc>-db must be a leaf STRICTLY BELOW its consumer (CNPG-DB rule)",
)
# cycle detection.
try:
compute_waves([
{"name": "p", "source": {"image": "x:1"}, "dependencies": [{"to": "q", "port": 1}]},
{"name": "q", "source": {"image": "x:1"}, "dependencies": [{"to": "p", "port": 1}]},
])
check(False, "compute_waves must raise on a dependency cycle")
except ValueError:
pass
# 7a) plan_apps_waves: sidecars inherit their unit's wave; not-onboarded SDs are skipped;
# EXTRA_UNITS anchor strictly above; a file absent from the tree is never placed.
present = {
"a.yaml", "svc.yaml", "svc-bootstrap-secret.sops.yaml",
"svc-db.yaml", "svc-db-app.sops.yaml", "gitea.yaml",
"gitea-actions.yaml", "gitea-actions-runner-secret.sops.yaml",
}
plan_sds = wsds + [{"name": "gitea", "source": {"image": "x:1"}},
{"name": "notonboarded", "source": {"image": "x:1"}}]
plan = plan_apps_waves(plan_sds, present)
flat = {f: wv for wv, fs in plan.items() for f in fs}
check(flat.get("a.yaml") == 0, "leaf app must land in wave 0")
check(flat.get("svc-db.yaml") == 0 and flat.get("svc-db-app.sops.yaml") == 0,
"CNPG DB manifest + its owner secret must be in the DB's (lower) wave")
check(flat.get("svc.yaml") == 1 and flat.get("svc-bootstrap-secret.sops.yaml") == 1,
"a DB-backed app + its bootstrap secret must be in the consumer wave, above the DB")
check(flat.get("gitea.yaml") == 0, "gitea (leaf) must be wave 0")
check(flat.get("gitea-actions.yaml") == 1
and flat.get("gitea-actions-runner-secret.sops.yaml") == 1,
"gitea-actions (EXTRA_UNITS, above=True) must be one wave above its anchor gitea")
check("notonboarded.yaml" not in flat,
"a catalog SD with no committed manifest must NOT be placed (not onboarded)")
check("b.yaml" not in flat, "a file absent from the tree must never be placed")
# 7b) generated Flux Kustomization CR wiring (dependsOn chain + prune/wait/path).
import yaml as _yaml # already imported at module top; local alias for clarity
fw0 = _yaml.safe_load(flux_wave_kustomization_text(0))
fw2 = _yaml.safe_load(flux_wave_kustomization_text(2))
check(fw0["metadata"]["name"] == "apps-wave-0"
and fw0["spec"]["dependsOn"] == [{"name": "platform-config"}]
and fw0["spec"]["path"] == "./apps/wave-0",
"wave 0 Flux Kustomization must dependsOn platform-config and point at ./apps/wave-0")
check(fw2["spec"]["dependsOn"] == [{"name": "apps-wave-1"}]
and fw2["spec"]["prune"] is True and fw2["spec"]["wait"] is True,
"wave N Flux Kustomization must dependsOn wave N-1 with prune+wait (health-gated)")
kt = _yaml.safe_load(wave_kustomization_text(1, ["svc.yaml", "sonarr.yaml"]))
check(kt["kind"] == "Kustomization" and kt["resources"] == ["svc.yaml", "sonarr.yaml"],
"wave kustomization must list exactly the wave's resources")
if failures:
for f in failures:
sys.stderr.write(f"SELFTEST FAIL: {f}\n")
@@ -1204,30 +1505,72 @@ def main(argv=None) -> int:
ap.add_argument("--no-image-automation", action="store_true",
help="Omit Flux ImageRepository/ImagePolicy CRs (for onboarding before the "
"image-reflector/-automation controllers are installed)")
ap.add_argument("--wave-layout", action="store_true",
help="With -o: place each rendered file under <out>/wave-<N>/<name>.yaml, "
"N = its dependency-ordered wave (compute_waves over the catalog)")
ap.add_argument("--emit-waves", action="store_true",
help="Generate the per-wave kustomize + Flux Kustomization CRs from the "
"catalog DAG and the committed apps tree (no manifest is rewritten)")
ap.add_argument("--print-waves", action="store_true",
help="Print the computed wave table (node -> wave) for the catalog and exit")
ap.add_argument("--catalog", default="catalog",
help="Catalog dir for --emit-waves/--print-waves/--wave-layout (default: catalog)")
ap.add_argument("--apps-dir", default="apps",
help="App-tier dir for --emit-waves (default: apps)")
ap.add_argument("--clusters-dir", default="clusters/base",
help="Dir to write clusters/base/apps-wave-<N>.yaml (default: clusters/base)")
args = ap.parse_args(argv)
if args.selftest:
return _selftest()
if args.print_waves:
waves = compute_waves(_load_catalog(Path(args.catalog)))
for node in sorted(waves, key=lambda n: (waves[n], n)):
sys.stdout.write(f"wave {waves[node]} {node}\n")
return 0
if args.emit_waves:
plan = emit_waves(Path(args.catalog), Path(args.apps_dir), Path(args.clusters_dir))
for wave in sorted(plan):
sys.stderr.write(f"[emit-waves] wave {wave}: {', '.join(plan[wave])}\n")
sys.stderr.write(
f"[emit-waves] wrote {len(plan)} wave kustomization(s) under {args.apps_dir}/ "
f"and clusters CRs under {args.clusters_dir}/\n"
)
return 0
image_automation = not args.no_image_automation
def _out_path(outdir: Path, name: str, sds: list) -> Path:
if args.wave_layout:
wdir = outdir / f"wave-{service_wave(sds, name)}"
wdir.mkdir(parents=True, exist_ok=True)
return wdir / f"{name}.yaml"
outdir.mkdir(parents=True, exist_ok=True)
return outdir / f"{name}.yaml"
if args.all:
src = Path(args.all)
files = sorted(p for p in src.glob("*.yaml") if p.name != "render.py")
sds = _load_catalog(src) if args.wave_layout else []
for f in files:
text = render_file(f, image_automation=image_automation)
if args.out_dir:
outdir = Path(args.out_dir)
outdir.mkdir(parents=True, exist_ok=True)
name = load_sd(f)["name"]
(outdir / f"{name}.yaml").write_text(text)
_out_path(Path(args.out_dir), load_sd(f)["name"], sds).write_text(text)
else:
sys.stdout.write(text)
return 0
if not args.input:
ap.error("provide a ServiceDefinition file, --all DIR, or --selftest")
sys.stdout.write(render_file(Path(args.input), image_automation=image_automation))
ap.error("provide a ServiceDefinition file, --all DIR, --emit-waves, or --selftest")
inp = Path(args.input)
text = render_file(inp, image_automation=image_automation)
if args.out_dir:
sds = _load_catalog(Path(args.catalog)) if args.wave_layout else []
_out_path(Path(args.out_dir), load_sd(inp)["name"], sds).write_text(text)
else:
sys.stdout.write(text)
return 0