diff --git a/bootstrap/flux/bootstrap.sh b/bootstrap/flux/bootstrap.sh index 1732765..580d20f 100755 --- a/bootstrap/flux/bootstrap.sh +++ b/bootstrap/flux/bootstrap.sh @@ -11,7 +11,17 @@ # WHAT THIS DOES (Phase 1, bootstrap-source method — reconciles from the LOCAL repo, no remote # git host exists yet per spec docs/05 + CONTRIBUTING.md): # -# 1. `flux install` — deploy the Flux controllers ONLY (source, kustomize, helm controllers). +# 1. `flux install` — deploy the Flux controllers. Besides the default set (source, kustomize, +# helm, notification) we ALSO install the two IMAGE controllers via +# `--components-extra` (see FLUX_COMPONENTS_EXTRA below): image-reflector- +# controller + image-automation-controller. These register the +# image.toolkit.fluxcd.io CRDs (ImageRepository/ImagePolicy/ImageUpdateAutomation) +# that Phase-4 hands-off updates need (ADR-0009, R-UPD-02). They are part of the +# flux-system install exactly like the other 4 controllers — NOT a reconciled +# HelmRelease — so their CRDs pre-exist before the first reconcile (this is why +# platform/image-automation can carry the ImageUpdateAutomation CR without the +# F-MON1 controllers/configs split: the CRD is present at bootstrap, like +# GitRepository/Kustomization are). WS-UPDATE (T-4.5). # We do NOT use `flux bootstrap git` in Phase 1 because that command commits # and OWNS its own gotk-sync GitRepository+Kustomization named `flux-system`. # Our entrypoints are ALREADY authored in clusters/base (flux-system.yaml = @@ -42,6 +52,9 @@ set -euo pipefail FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}" # dev is the ONLY Phase-1 build target FLUX_NAMESPACE="${FLUX_NAMESPACE:-flux-system}" SOPS_SECRET_NAME="${SOPS_SECRET_NAME:-sops-age}" +# The two Phase-4 image-automation controllers are installed alongside the default set so their CRDs +# exist before reconcile (ADR-0009, R-UPD-02). Override to "" to build a cluster without them. +FLUX_COMPONENTS_EXTRA="${FLUX_COMPONENTS_EXTRA:-image-reflector-controller,image-automation-controller}" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || cd "${script_dir}/../.." && pwd)" @@ -54,8 +67,16 @@ for bin in flux kubectl; do done # --- 1. install Flux controllers --------------------------------------------- -echo "==> [1/3] flux install (controllers only)" -flux install --namespace="${FLUX_NAMESPACE}" +# Default controllers + the image-reflector/image-automation pair (Phase-4 hands-off updates, +# ADR-0009). `--components-extra` regenerates flux-system with all controllers and registers the +# image.toolkit.fluxcd.io CRDs before any reconcile — so platform/image-automation's CR applies cleanly. +if [ -n "${FLUX_COMPONENTS_EXTRA}" ]; then + echo "==> [1/3] flux install (default controllers + extra: ${FLUX_COMPONENTS_EXTRA})" + flux install --namespace="${FLUX_NAMESPACE}" --components-extra="${FLUX_COMPONENTS_EXTRA}" +else + echo "==> [1/3] flux install (default controllers only)" + flux install --namespace="${FLUX_NAMESPACE}" +fi # --- 2. verify the out-of-band age key Secret exists ------------------------- # We DO NOT create it here (it holds the private key). The operator installs it per diff --git a/bootstrap/sops/secret-templates/gitea-image-automation-credentials.yaml.example b/bootstrap/sops/secret-templates/gitea-image-automation-credentials.yaml.example new file mode 100644 index 0000000..58f25c3 --- /dev/null +++ b/bootstrap/sops/secret-templates/gitea-image-automation-credentials.yaml.example @@ -0,0 +1,35 @@ +# ============================================================================= +# TEMPLATE — NOT APPLIED, NOT ENCRYPTED (.yaml.example is ignored by kustomize +# and never auto-encrypted by SOPS). Reference scaffold only. +# +# PURPOSE: the git WRITE credential Flux image-automation uses to commit machine-authored image tag +# advances back to the GitOps source (ADR-0009, R-UPD-02). image-automation-controller has NO secretRef +# of its own — it reuses the `flux-system` GitRepository's `.spec.secretRef`. So this Secret is wired +# onto the flux-system GitRepository, NOT onto the ImageUpdateAutomation. +# +# (a) Generate a Gitea access token (scope: write:repository) for the bootstrap bot user, e.g.: +# curl -s -u -X POST http://10.73.0.1:3000/api/v1/users/fluxcdbot/tokens \ +# -H 'content-type: application/json' \ +# -d '{"name":"image-automation","scopes":["write:repository"]}' +# (the operator's bash step; store the returned token straight into this file — never print it). +# (b) Copy to: platform/image-automation/gitea-image-automation-credentials.sops.yaml +# add under `resources:` in platform/image-automation/kustomization.yaml: +# - gitea-image-automation-credentials.sops.yaml +# (c) Encrypt: sops --encrypt --in-place platform/image-automation/gitea-image-automation-credentials.sops.yaml +# (d) Consume it: attach to the flux-system GitRepository via the bootstrap source-url patch +# (bootstrap/flux/patches/flux-system-source-url.yaml) by adding: +# spec: +# secretRef: +# name: gitea-image-automation-credentials +# NOTE: the Secret must live in the flux-system namespace (same as the GitRepository). +# Basic-auth keys are `username` + `password` (Gitea accepts the token as password). +# ============================================================================= +apiVersion: v1 +kind: Secret +metadata: + name: gitea-image-automation-credentials + namespace: flux-system +type: Opaque +stringData: + username: fluxcdbot + password: REPLACE-ME # Gitea access token with write:repository scope (NOT a login password) diff --git a/platform/image-automation/README.md b/platform/image-automation/README.md index 58f707e..864e1cd 100644 --- a/platform/image-automation/README.md +++ b/platform/image-automation/README.md @@ -8,22 +8,69 @@ - **Serves:** R-UPD-02 (with R-UPD-01/03 safety rails) - **Owner:** WS-UPDATE (mechanism), WS-GITOPS (policy rendering from ServiceDefinition) -## What this stub is +## What lives here -The **cluster-wide** `ImageUpdateAutomation` that scans registries and **commits the resolved version -back to git** (machine-authored, auditable — ADR-0009). No human or AI authors a tag bump (R-UPD-02). +- **`image-update-automation.yaml`** — the **cluster-wide** `ImageUpdateAutomation` that scans + registries per each service's `ImagePolicy` and **commits the resolved stable version back to git** + (machine-authored, auditable — ADR-0009). No human or AI authors a tag bump (R-UPD-02). +- **`per-service/`** — the **render contract** (`update` block → `ImageRepository`/`ImagePolicy`/setter + marker) plus a worked example (`pihole.example.yaml`). **Reference only, not deployed** — WS-GITOPS + emits the equivalent next to each real app's Deployment/HelmRelease under `apps/`. + +## Controllers (how the CRDs get here) + +The two controllers — **image-reflector-controller** + **image-automation-controller** — are installed +as part of the `flux-system` install by **`bootstrap/flux/bootstrap.sh`** via +`flux install --components-extra=image-reflector-controller,image-automation-controller`, exactly like +the other four Flux controllers. Consequences: + +- The `image.toolkit.fluxcd.io` CRDs (`ImageRepository`/`ImagePolicy`/`ImageUpdateAutomation`) **exist + before the first reconcile** — just like `GitRepository`/`Kustomization` do. +- So the `ImageUpdateAutomation` CR here needs **no** F-MON1 controllers/configs split: F-MON1 bites + only when a CR is applied in the *same* Flux pass that installs its CRD (via a HelmRelease). Here the + CRD is a bootstrap artifact, so the CR applies cleanly from the `platform` Kustomization. (The + per-service CRs likewise ride with `apps/`, which `dependsOn platform`.) ## Scope boundary (contract) -This dir holds **only** the cluster-wide automation + git write-back settings. **Per-service** -`ImageRepository` / `ImagePolicy` (semver range / stable channel) are rendered from each -ServiceDefinition's `update` block by WS-GITOPS/WS-UPDATE and live with `apps/` — the "Update policy -contract" in `roadmap/workstreams.md`. **Helm chart / manifest bumps** are Renovate's job (separate), -not this controller's. +- **This dir:** the cluster-wide automation + git write-back settings. +- **`apps//`:** per-service `ImageRepository` / `ImagePolicy` (semver stable range) + the Deployment + setter marker — rendered from each ServiceDefinition's `update` block (see `per-service/README.md`). +- **Renovate (`/renovate.json`):** Helm **chart** / dependency / operator-manifest bumps (opens PRs, + auto-merges allowed classes on green CI). Not this controller's job, and deliberately non-overlapping + (Renovate's docker/kubernetes image managers are disabled so it never fights Flux over a tag). -## TODO (later phases — Phase 4) +## Safety rails (acceptance gates — ADR-0009) -- Confirm write-back strategy (direct commit to tracked branch vs. Renovate-style PRs). -- Wire the safety rails that are **acceptance gates** (ADR-0009): stable-only, health-gated rollout, - auto-rollback on failed health, `pin` escape hatch, alert on update failure. -- Requires the Flux **image-reflector** + **image-automation** controllers (Flux bootstrap component). +| Rail | Where enforced | +|---|---| +| **Stable-only** (R-UPD-01) | `ImagePolicy` semver range + a `filterTags` pattern that admits only bare `X.Y.Z` (excludes `-rc`/`latest`). | +| **No human/AI authoring** (R-UPD-02) | image-automation writes the tag at the `$imagepolicy` setter; the commit is machine-authored (`fluxcdbot`). | +| **Health-gated auto-rollback** (R-UPD-03) | HelmRelease `upgrade.remediation` + `rollback` (preferred), or Deployment `progressDeadlineSeconds` + readiness probe. | +| **Pin escape hatch** (R-UPD-03) | `update.pin` renders a fixed image with **no** setter marker and no/suspended `ImagePolicy` — automation is frozen. | +| **Alert on failure** (R-UPD-02 + R-OBS) | `platform/monitoring/alerts.yaml` group `platform-health.automated-updates` (`ImageUpdateAutomationFailing`, `ImagePolicyStalled`, `UpdateHealthGateFailed`). | + +Proven by the L3 conformance test +`conformance/tests/l3-fault-injection/hands-off-update/` (auto-advance stable, bad-update +auto-rollback + alert-wired, pin holds). + +## Operator hand-off: git write credential (required before enabling) + +image-automation-controller reuses the **`flux-system` GitRepository's `.spec.secretRef`** for push auth +(it has no independent secretRef). The Phase-1 source is anonymous/read-only, so before write-back works +the operator must: + +1. Create a Gitea access token (scope `write:repository`) for the bot and put it in the SOPS secret + from the template `bootstrap/sops/secret-templates/gitea-image-automation-credentials.yaml.example` + → `platform/image-automation/gitea-image-automation-credentials.sops.yaml` (name + `gitea-image-automation-credentials`, keys `username`/`password`), then `sops --encrypt --in-place`. +2. Add that file to this dir's `kustomization.yaml` `resources:`. +3. Attach it to the source: add `spec.secretRef.name: gitea-image-automation-credentials` to the + `flux-system` GitRepository via `bootstrap/flux/patches/flux-system-source-url.yaml`. + +## Enabling (deferred — like velero/oauth2-proxy) + +`image-automation` is intentionally **commented out** in `platform/kustomization.yaml`. Enable it only +after the write credential above is provisioned — otherwise the `ImageUpdateAutomation` reconcile errors +on push and, under the unified graph's `platform` wait, could stall the downstream graph. Uncomment the +`- image-automation` line once the hand-off is done. diff --git a/platform/image-automation/image-update-automation.yaml b/platform/image-automation/image-update-automation.yaml index fa48602..210a228 100644 --- a/platform/image-automation/image-update-automation.yaml +++ b/platform/image-automation/image-update-automation.yaml @@ -1,39 +1,57 @@ --- # Flux image-update-automation — the CLUSTER-WIDE write-back engine for hands-off image tag advances -# (ADR-0009, R-UPD-02). It scans registries per each service's ImagePolicy and commits the resolved -# version back to git — no human/AI authors the bump. Phase 0 STUB. +# (ADR-0009, R-UPD-02). image-reflector-controller scans each per-service ImageRepository on its +# interval; image-automation-controller resolves each ImagePolicy and writes the advanced tag back to +# git at the `# {"$imagepolicy": "..."}` setter markers. The commit is MACHINE-authored under the +# operator's once-declared policy — no human/AI edits a tag (R-UPD-01/02). # -# SCOPE: this dir holds ONLY the cluster-wide ImageUpdateAutomation + git write-back settings. -# Per-service `ImageRepository`/`ImagePolicy` are rendered from each ServiceDefinition's `update` -# block (WS-GITOPS/WS-UPDATE) and live with apps/ — NOT here. (Contract: roadmap/workstreams.md -# "Update policy contract".) +# SCOPE (contract, roadmap/workstreams.md "Update policy contract"): this dir holds ONLY the +# cluster-wide ImageUpdateAutomation + git write-back settings. PER-SERVICE ImageRepository/ImagePolicy +# are rendered from each ServiceDefinition's `update` block (renderer-contract §update) and live NEXT +# TO each app's Deployment/HelmRelease under apps/ — see ./per-service/ for the render contract + a +# worked example. Helm CHART / dependency bumps are Renovate's job (renovate.json), not this controller. +# +# CONTROLLERS: image-reflector-controller + image-automation-controller are installed as part of the +# flux-system install (bootstrap/flux/bootstrap.sh `--components-extra`), so the image.toolkit CRDs +# pre-exist before reconcile — this CR applies cleanly without the F-MON1 controllers/configs split +# (the CRD is present at bootstrap, exactly like GitRepository/Kustomization are). See README.md. apiVersion: image.toolkit.fluxcd.io/v1beta2 kind: ImageUpdateAutomation metadata: name: flux-image-automation namespace: flux-system spec: - interval: 30m # the "schedule" on which stable versions are picked up (R-UPD-02) + # The scan/apply schedule (R-UPD-02). image-reflector scans registries continuously per + # ImageRepository.spec.interval; this interval bounds how often resolved tags are written back. + interval: 30m sourceRef: kind: GitRepository name: flux-system + # ^ Write-back auth REUSES this GitRepository's `.spec.secretRef` (image-automation-controller has + # no independent secretRef). The Phase-1 flux-system source is currently anonymous/read-only + # (public host-Gitea clone). For write-back the operator must attach a WRITE credential to the + # flux-system GitRepository — see README.md "Operator hand-off: git write credential" and the + # SOPS template bootstrap/sops/secret-templates/gitea-image-automation-credentials.yaml.example. git: checkout: ref: - branch: main # TODO(WS-UPDATE): confirm write-back branch (may open PRs instead) + branch: main # reconciled branch (matches clusters/base/flux-system.yaml) commit: author: - name: fluxcdbot # machine author — the auditable, intended write-back (ADR-0009) + # Machine author — the auditable, intended write-back (ADR-0009). No human/AI authors the bump. + name: fluxcdbot email: fluxcdbot@users.noreply.home.keithjackson.dev messageTemplate: | - chore(images): automated update + chore(images): automated stable update [ci skip] - Machine-authored image tag advance per ImagePolicy (R-UPD-02, ADR-0009). + Machine-authored image tag advance under the declared per-service ImagePolicy + (R-UPD-01 stable-only, R-UPD-02 no human/AI authoring). ADR-0009. + {{ range .Changed.Changes -}} + - {{ .OldValue }} -> {{ .NewValue }} ({{ .Setter }}) + {{ end -}} push: - branch: main + branch: main # push straight to the tracked branch (default write-back strategy, + # ADR-0009). Renovate handles PR-style bumps for charts/deps separately. update: - strategy: Setters # image-automation setter markers in per-service manifests (apps/) - path: ./apps -# TODO(WS-UPDATE, Phase 4): safety rails are acceptance gates (ADR-0009) — stable-only, health-gated -# rollout, auto-rollback on failed health, pin escape hatch, alert on update failure. Prove a test -# service auto-advances with no human/AI edit; a bad update auto-rolls-back + alerts (R-UPD-01/02/03). + strategy: Setters # advance the `$imagepolicy` setter markers in per-service manifests + path: ./apps # only apps/ carries per-service image markers (platform charts = Renovate)