# promtool rule-unit tests for FluxRolloutInProgress / FluxRolloutStuck — the deploy-notification # pair in the platform-health.flux group (platform/monitoring/alerts.yaml). # Run with: platform/monitoring/tests/run.sh (the merge gate runs the same harness via # conformance/tests/l0-static/alert-rules-unit-tested) # # WHY THESE TESTS CARRY MORE WEIGHT THAN USUAL. This pair is a NOTIFICATION, so both failure # directions are silent in production and neither is visible on a healthy cluster: # * fire when it should not -> a permanently-firing "deploy in progress" that mails on every # repeat_interval and trains the operator to ignore Flux mail. Shipped once already in this # repo (CNPG, 2026-09-12). # * fail to fire -> no deploy notification at all, which is the exact complaint this pair was # written to answer. Indistinguishable from "no deploys happened". # So every clause is pinned in both directions here, and the two GUARDS in the expression (the # missing-exporter guard and the suspended-Kustomization exclusion) each get a test whose ONLY # job is to fail if that clause is deleted. # # MODELLING A REVISION CHANGE. `revision` is a LABEL, so a Kustomization moving from revision A to # revision B is modelled as series A ending and series B beginning — not as one series changing # value. Prometheus' 5m lookback then means series A is still resolvable for 5m after its last # sample, so every eval_time below is placed at least 5m clear of a handoff. That is deliberate: # it keeps these tests about the RULE and not about staleness semantics. # # Sample interval 1m; the longest `for` under test is 30m. rule_files: - generated.rules.yaml evaluation_interval: 1m tests: # ============================================================================================= # 1. CONVERGED — the steady state this cluster is in almost all of the time. # Both rules MUST be silent. This is the "not permanently firing" half of the proof, and it # is the test that fails if `unless on (revision)` is mis-spelled: the left-hand side is # non-empty here, and only the revision match suppresses it. # ============================================================================================= - interval: 1m name: converged (every Kustomization on the GitRepository revision) - silent input_series: - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:aaaaaaa",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="platform",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x40' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x40' alert_rule_test: - eval_time: 35m alertname: FluxRolloutInProgress exp_alerts: [] - eval_time: 35m alertname: FluxRolloutStuck exp_alerts: [] # ============================================================================================= # 2. A COMMIT LANDS — the notification this whole change exists to produce. # GitRepository moves to revision bbbbbbb at t=11m. apps-wave-0 catches up at t=14m; # `platform` never does. # t=25m -> FluxRolloutInProgress fires for `platform` ONLY (apps-wave-0 has converged), # and FluxRolloutStuck is still quiet because its 30m has not elapsed. # t=50m -> both fire. ("started and never finished" is a real, separate condition.) # ============================================================================================= - interval: 1m name: new revision on main - InProgress fires for the lagging Kustomization only input_series: # The old artifact revision ends at t=10m, the new one starts at t=11m. - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:aaaaaaa",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '1x10' - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:bbbbbbb",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '_x10 1x70' # `platform` stays on the old revision for the whole test. - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x80' # apps-wave-0 catches up at t=14m. - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x8' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",revision="main@sha1:bbbbbbb",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '_x13 1x67' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="platform",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x80' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x80' alert_rule_test: # The deploy notification, ~14m after the commit was fetched. Exactly ONE alert: a rule that # ignored the revision match would also list apps-wave-0, which has already converged. - eval_time: 25m alertname: FluxRolloutInProgress exp_alerts: - exp_labels: notification: deploy job: kube-state-metrics namespace: flux-system name: platform revision: "main@sha1:aaaaaaa" customresource_group: kustomize.toolkit.fluxcd.io customresource_kind: Kustomization customresource_version: v1 exp_annotations: summary: "Flux is applying a new revision — platform is behind main" description: "Kustomization flux-system/platform has applied main@sha1:aaaaaaa, which is not the revision the flux-system GitRepository has fetched from main. THIS MAIL MEANS A GIT CHANGE STARTED ROLLING OUT; the RESOLVED mail for this same alert means every Kustomization reached the new revision and the rollout is complete." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-rollout" # `for: 30m` on the escalation is real — not yet firing 14m in. - eval_time: 25m alertname: FluxRolloutStuck exp_alerts: [] # ...and it does fire once the rollout has genuinely hung. - eval_time: 50m alertname: FluxRolloutStuck exp_alerts: - exp_labels: severity: warning job: kube-state-metrics namespace: flux-system name: platform revision: "main@sha1:aaaaaaa" customresource_group: kustomize.toolkit.fluxcd.io customresource_kind: Kustomization customresource_version: v1 exp_annotations: summary: "Flux rollout has not completed: platform stuck on an old revision" description: "Kustomization flux-system/platform has been behind the flux-system GitRepository revision for 30m (applied main@sha1:aaaaaaa) — a git change started rolling out and has not converged. Check the Kustomization's Ready condition and its dependencies." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-rollout" # ============================================================================================= # 3. THE ROLLOUT COMPLETES -> the alert clears, which is what sends the "applied" mail. # Without this test the completion half of the feature is unproven: an expression that fired # correctly but never emptied itself would pass test 2 and still never deliver a resolved # notification. # ============================================================================================= - interval: 1m name: rollout completes - InProgress clears (this is the resolved/completion notification) input_series: - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:aaaaaaa",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '1x10' - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:bbbbbbb",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '_x10 1x70' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x20' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:bbbbbbb",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '_x23 1x57' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="platform",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x80' alert_rule_test: # Mid-rollout: firing. - eval_time: 20m alertname: FluxRolloutInProgress exp_alerts: - exp_labels: notification: deploy job: kube-state-metrics namespace: flux-system name: platform revision: "main@sha1:aaaaaaa" customresource_group: kustomize.toolkit.fluxcd.io customresource_kind: Kustomization customresource_version: v1 exp_annotations: summary: "Flux is applying a new revision — platform is behind main" description: "Kustomization flux-system/platform has applied main@sha1:aaaaaaa, which is not the revision the flux-system GitRepository has fetched from main. THIS MAIL MEANS A GIT CHANGE STARTED ROLLING OUT; the RESOLVED mail for this same alert means every Kustomization reached the new revision and the rollout is complete." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-rollout" # Converged (5m clear of the old series' last sample at t=20m): silent, and it never had # 30m of continuous firing, so the escalation must not have fired either. - eval_time: 32m alertname: FluxRolloutInProgress exp_alerts: [] - eval_time: 32m alertname: FluxRolloutStuck exp_alerts: [] # ============================================================================================= # 4. GUARD 1 — kube_flux_gitrepository_info ABSENT (KSM rolled without the ConfigMap, the flux # RBAC was lost, customResourceState reverted). `unless on (revision) ` removes # NOTHING, so without the `and on () (count(...) > 0)` guard EVERY Kustomization alerts # forever. This test exists to go red the moment that guard is deleted. # Correct behaviour is SILENCE: a broken exporter is FluxReconciliationFailure's job and the # platform-health.monitoring-self rules', not a deploy notification's. # ============================================================================================= - interval: 1m name: GUARD - GitRepository metric absent - silent, NOT 16 alerts forever input_series: - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x60' - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x60' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="platform",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x60' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="apps-wave-0",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x60' promql_expr_test: # Stated explicitly so the reason for the silence is unambiguous in the test output: the # guard's own count is empty, not merely false. - expr: 'count(kube_flux_gitrepository_info{name="flux-system", revision!=""})' eval_time: 45m exp_samples: [] alert_rule_test: - eval_time: 45m alertname: FluxRolloutInProgress exp_alerts: [] - eval_time: 45m alertname: FluxRolloutStuck exp_alerts: [] # ============================================================================================= # 5. GUARD 2 — a SUSPENDED Kustomization pins an old revision by design. It must not hold the # rollout notification open (or the "applied" mail never arrives), while a non-suspended # Kustomization on the same old revision MUST still alert. # Both halves are asserted in one test on purpose: an `unless` written with the wrong labels # would suppress EVERYTHING and would pass a suppression-only test. # ============================================================================================= - interval: 1m name: GUARD - suspended Kustomization excluded, non-suspended one still alerts input_series: - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:bbbbbbb",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '1x40' # Suspended and behind -> excluded. - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="platform",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="platform",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' # Not suspended and behind -> must alert. - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="conformance",revision="main@sha1:aaaaaaa",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="conformance",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x40' alert_rule_test: - eval_time: 20m alertname: FluxRolloutInProgress exp_alerts: - exp_labels: notification: deploy job: kube-state-metrics namespace: flux-system name: conformance revision: "main@sha1:aaaaaaa" customresource_group: kustomize.toolkit.fluxcd.io customresource_kind: Kustomization customresource_version: v1 exp_annotations: summary: "Flux is applying a new revision — conformance is behind main" description: "Kustomization flux-system/conformance has applied main@sha1:aaaaaaa, which is not the revision the flux-system GitRepository has fetched from main. THIS MAIL MEANS A GIT CHANGE STARTED ROLLING OUT; the RESOLVED mail for this same alert means every Kustomization reached the new revision and the rollout is complete." runbook_url: "https://${BASE_DOMAIN}/runbooks/flux-rollout" # ============================================================================================= # 6. A Kustomization that has NEVER applied anything publishes no `revision` label at all (KSM # omits absent paths). `revision!=""` must drop it rather than let it group under the empty # string and compare equal to the GitRepository revision. # ============================================================================================= - interval: 1m name: never-applied Kustomization (no revision label) is not counted as converged or lagging input_series: - series: 'kube_flux_gitrepository_info{job="kube-state-metrics",namespace="flux-system",name="flux-system",revision="main@sha1:bbbbbbb",customresource_group="source.toolkit.fluxcd.io",customresource_kind="GitRepository",customresource_version="v1"}' values: '1x40' # No `revision` label at all — this is what KSM publishes for status.lastAppliedRevision="". - series: 'kube_flux_kustomization_info{job="kube-state-metrics",namespace="flux-system",name="brand-new",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '1x40' - series: 'kube_flux_kustomization_suspended{job="kube-state-metrics",namespace="flux-system",name="brand-new",customresource_group="kustomize.toolkit.fluxcd.io",customresource_kind="Kustomization",customresource_version="v1"}' values: '0x40' alert_rule_test: # Silent: FluxReconciliationFailure (Ready=False for 15m) is what covers a Kustomization # that cannot apply at all. This pair only ever speaks about revisions it can compare. - eval_time: 30m alertname: FluxRolloutInProgress exp_alerts: [] - eval_time: 30m alertname: FluxRolloutStuck exp_alerts: []