home-network-k3s-impl/conformance/tests/l0-static/vuln-alerts-unit-tested/check.sh

#!/usr/bin/env bash
# =============================================================================
# L0 static check (offline, no cluster) — vuln-alerts-unit-tested
# Serves: R-VULN-06 (docs/17 "Acceptance gates": alerts unit-tested, including the absent-metric cases).
#
# GIVEN platform/vuln-scanner/alerts.yaml and alerts.promtool-test.yaml,
# THEN  (1) the rules are exactly the five contract alerts, with the two absent()-bearing rules;
#       (2) each alert is asserted FIRING in a promtool test;
#       (3) VulnScanStale and VulnTriageBotSilent each fire in a test with NO input series for their
#           own metric (the absent case) and in one where it vanishes (re-fire);
#       (4) each alert carries severity and a runbook_url;
#       (5) when promtool is present (the toolbox), the actual harness passes on this file;
#   AND (6) the evaluator is run on mutated copies (the absent arm deleted, a firing assertion
#           removed, an alert dropped) and must flag each (non-vacuity).
# =============================================================================
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "${DIR}/../../../.." && pwd)"
exec python3 - "$ROOT" <<'PY'
import copy, re, shutil, subprocess, sys
from pathlib import Path
import yaml

ROOT = Path(sys.argv[1])
NAME = "vuln-alerts-unit-tested"
FAILURES, CHECKS = [], 0
ALERTS = {"VulnScanStale", "VulnDatabaseStale", "VulnCoverageGap", "VulnCriticalFixAvailable", "VulnTriageBotSilent"}
ABSENT_ALERTS = {
    "VulnScanStale": "vuln_scanner_last_complete_run_timestamp_seconds",
    "VulnTriageBotSilent": "vuln_scanner_triage_heartbeat_timestamp_seconds",
}


def check(cond, desc):
    global CHECKS
    CHECKS += 1
    if not cond:
        FAILURES.append(desc)


COMP = ROOT / "platform" / "vuln-scanner"
for f in ("alerts.yaml", "alerts.promtool-test.yaml"):
    if not (COMP / f).is_file():
        print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/{f} is absent", file=sys.stderr)
        raise SystemExit(1)
rules = yaml.safe_load((COMP / "alerts.yaml").read_text())
tests = yaml.safe_load((COMP / "alerts.promtool-test.yaml").read_text())


def alert_rules(doc):
    return {r["alert"]: r for g in doc["spec"]["groups"] for r in g["rules"]}


def eval_alerts(doc):
    bad = []
    got = alert_rules(doc)
    if set(got) != ALERTS:
        bad.append(f"alerts are {sorted(got)}, expected {sorted(ALERTS)}")
    for name, metric in ABSENT_ALERTS.items():
        if name in got and f"absent({metric})" not in got[name]["expr"].replace(" ", ""):
            bad.append(f"{name} lacks the absent({metric}) arm")
    for name, r in got.items():
        if not (r.get("labels") or {}).get("severity"):
            bad.append(f"{name} has no severity label")
        if "/runbooks/" not in ((r.get("annotations") or {}).get("runbook_url") or ""):
            bad.append(f"{name} has no runbook_url")
    return bad


def firing_assertions(test):
    """(alertname -> assertion list) for the firing expectations of one promtool test."""
    out = {}
    for pe in test.get("promql_expr_test") or []:
        expr, samples = pe.get("expr", ""), pe.get("exp_samples") or []
        m = re.search(r'alertname="(Vuln\w+)"', expr)
        if m and samples and 'alertstate="firing"' in (expr + " " + " ".join(s.get("labels", "") for s in samples)):
            out.setdefault(m.group(1), []).append(pe)
    return out


def series_for(test):
    return [s.get("series", "") for s in test.get("input_series") or []]


def eval_tests(doc):
    bad = []
    all_tests = doc.get("tests") or []
    fired = set()
    for t in all_tests:
        fired |= set(firing_assertions(t))
    missing = ALERTS - fired
    if missing:
        bad.append(f"no firing promtool assertion for {sorted(missing)}")
    for alert, metric in ABSENT_ALERTS.items():
        absent_case = any(
            alert in firing_assertions(t) and not any(s.startswith(metric) for s in series_for(t))
            for t in all_tests)
        if not absent_case:
            bad.append(f"{alert} has no firing test with NO {metric} series (the absent case)")
        vanish_case = any(
            alert in firing_assertions(t) and any(s.startswith(metric) and "stale" in str(i.get("values", ""))
                                                  for i in t.get("input_series", []) for s in [i.get("series", "")])
            for t in all_tests)
        if not vanish_case:
            bad.append(f"{alert} has no test where its metric goes stale (the re-fire case)")
    return bad


for v in eval_alerts(rules):
    check(False, f"(1,4) {v}")
for v in eval_tests(tests):
    check(False, f"(2,3) {v}")
check(len(alert_rules(rules)) == 5, "(1) five alert rules parsed")

# (5) the real harness, only when the rule engine is present.
if shutil.which("promtool"):
    rc = subprocess.run(["bash", str(ROOT / "platform/monitoring/tests/run.sh"),
                         "platform/vuln-scanner/alerts.promtool-test.yaml"],
                        cwd=str(ROOT), capture_output=True, text=True)
    check(rc.returncode == 0, f"(5) the promtool harness failed: rc={rc.returncode}\n{rc.stdout[-800:]}{rc.stderr[-800:]}")

# (6) non-vacuity
m = copy.deepcopy(rules)
for g in m["spec"]["groups"]:
    for r in g["rules"]:
        if r["alert"] == "VulnTriageBotSilent":
            r["expr"] = r["expr"].split(" or ")[0]
check(bool(eval_alerts(m)), "(6) evaluator missed: absent arm deleted")
m = copy.deepcopy(rules)
m["spec"]["groups"][0]["rules"].pop(1)
check(bool(eval_alerts(m)), "(6) evaluator missed: alert dropped")
m = copy.deepcopy(rules)
for g in m["spec"]["groups"]:
    for r in g["rules"]:
        r["annotations"].pop("runbook_url", None)
check(bool(eval_alerts(m)), "(6) evaluator missed: runbook_url removed")

m = copy.deepcopy(tests)
for t in m["tests"]:
    t["promql_expr_test"] = [p for p in t["promql_expr_test"]
                             if 'VulnCoverageGap' not in p.get("expr", "")]
check(bool(eval_tests(m)), "(6) evaluator missed: firing assertion removed")
m = copy.deepcopy(tests)
m["tests"] = [t for t in m["tests"] if "ABSENT" not in t.get("name", "")]
check(bool(eval_tests(m)), "(6) evaluator missed: absent-case tests removed")

if FAILURES:
    print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
    for f in FAILURES:
        print(f"  - {f}", file=sys.stderr)
    raise SystemExit(1)
print(f"PASS ({NAME}): {CHECKS} assertions")
PY