feat(vuln): vuln-scanner platform manifests, Flux tier, alerts, L0/L2/L3 tests
Hand-authored platform/vuln-scanner (non-gating Flux tier, digest-pinned Trivy 0.76.1 and python 3.14.9-alpine, read-only RBAC, per-pod oauth2-proxy, internal-only cve.<domain>), five alerts with promtool tests, R-VULN-01..09 registry entries, and vuln-* L0 checks with mutation self-tests. src/ and the bot workflow are integrated separately. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
4c3a284cad
commit
b85d1e3ed7
47 files changed
+2572
-6
No files matched your search
@@ -0,0 +1,29 @@
|
||||
# =============================================================================
|
||||
# L0 STATIC — the five scanner alerts exist and are unit-tested, including the absent-metric cases
|
||||
# (R-VULN-06, docs/17 "Acceptance gates"). Offline, no cluster.
|
||||
# =============================================================================
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-alerts-unit-tested
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-06"
|
||||
conformance.home.k3s/layer: "L0"
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
conformance.home.k3s/executable: "true"
|
||||
conformance.home.k3s/intent: >-
|
||||
VulnScanStale, VulnDatabaseStale, VulnCoverageGap, VulnCriticalFixAvailable and
|
||||
VulnTriageBotSilent exist, each promtool-tested firing, and the two absent-metric rules fire
|
||||
in a test with no such series. Proven by mutation.
|
||||
spec:
|
||||
steps:
|
||||
- name: assert-alert-unit-tests
|
||||
description: >-
|
||||
Parse alerts.yaml and alerts.promtool-test.yaml, assert every alert and the two absent cases
|
||||
are pinned, run the harness when promtool is present, and run the evaluator on mutated copies.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# L0 static check (offline, no cluster) — vuln-alerts-unit-tested
|
||||
# Serves: R-VULN-06 (docs/17 "Acceptance gates": alerts unit-tested, including the absent-metric cases).
|
||||
#
|
||||
# GIVEN platform/vuln-scanner/alerts.yaml and alerts.promtool-test.yaml,
|
||||
# THEN (1) the rules are exactly the five contract alerts, with the two absent()-bearing rules;
|
||||
# (2) each alert is asserted FIRING in a promtool test;
|
||||
# (3) VulnScanStale and VulnTriageBotSilent each fire in a test with NO input series for their
|
||||
# own metric (the absent case) and in one where it vanishes (re-fire);
|
||||
# (4) each alert carries severity and a runbook_url;
|
||||
# (5) when promtool is present (the toolbox), the actual harness passes on this file;
|
||||
# AND (6) the evaluator is run on mutated copies (the absent arm deleted, a firing assertion
|
||||
# removed, an alert dropped) and must flag each (non-vacuity).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "${DIR}/../../../.." && pwd)"
|
||||
exec python3 - "$ROOT" <<'PY'
|
||||
import copy, re, shutil, subprocess, sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
ROOT = Path(sys.argv[1])
|
||||
NAME = "vuln-alerts-unit-tested"
|
||||
FAILURES, CHECKS = [], 0
|
||||
ALERTS = {"VulnScanStale", "VulnDatabaseStale", "VulnCoverageGap", "VulnCriticalFixAvailable", "VulnTriageBotSilent"}
|
||||
ABSENT_ALERTS = {
|
||||
"VulnScanStale": "vuln_scanner_last_complete_run_timestamp_seconds",
|
||||
"VulnTriageBotSilent": "vuln_scanner_triage_heartbeat_timestamp_seconds",
|
||||
}
|
||||
|
||||
|
||||
def check(cond, desc):
|
||||
global CHECKS
|
||||
CHECKS += 1
|
||||
if not cond:
|
||||
FAILURES.append(desc)
|
||||
|
||||
|
||||
COMP = ROOT / "platform" / "vuln-scanner"
|
||||
for f in ("alerts.yaml", "alerts.promtool-test.yaml"):
|
||||
if not (COMP / f).is_file():
|
||||
print(f"CHECK FAIL ({NAME}): platform/vuln-scanner/{f} is absent", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
rules = yaml.safe_load((COMP / "alerts.yaml").read_text())
|
||||
tests = yaml.safe_load((COMP / "alerts.promtool-test.yaml").read_text())
|
||||
|
||||
|
||||
def alert_rules(doc):
|
||||
return {r["alert"]: r for g in doc["spec"]["groups"] for r in g["rules"]}
|
||||
|
||||
|
||||
def eval_alerts(doc):
|
||||
bad = []
|
||||
got = alert_rules(doc)
|
||||
if set(got) != ALERTS:
|
||||
bad.append(f"alerts are {sorted(got)}, expected {sorted(ALERTS)}")
|
||||
for name, metric in ABSENT_ALERTS.items():
|
||||
if name in got and f"absent({metric})" not in got[name]["expr"].replace(" ", ""):
|
||||
bad.append(f"{name} lacks the absent({metric}) arm")
|
||||
for name, r in got.items():
|
||||
if not (r.get("labels") or {}).get("severity"):
|
||||
bad.append(f"{name} has no severity label")
|
||||
if "/runbooks/" not in ((r.get("annotations") or {}).get("runbook_url") or ""):
|
||||
bad.append(f"{name} has no runbook_url")
|
||||
return bad
|
||||
|
||||
|
||||
def firing_assertions(test):
|
||||
"""(alertname -> assertion list) for the firing expectations of one promtool test."""
|
||||
out = {}
|
||||
for pe in test.get("promql_expr_test") or []:
|
||||
expr, samples = pe.get("expr", ""), pe.get("exp_samples") or []
|
||||
m = re.search(r'alertname="(Vuln\w+)"', expr)
|
||||
if m and samples and 'alertstate="firing"' in (expr + " " + " ".join(s.get("labels", "") for s in samples)):
|
||||
out.setdefault(m.group(1), []).append(pe)
|
||||
return out
|
||||
|
||||
|
||||
def series_for(test):
|
||||
return [s.get("series", "") for s in test.get("input_series") or []]
|
||||
|
||||
|
||||
def eval_tests(doc):
|
||||
bad = []
|
||||
all_tests = doc.get("tests") or []
|
||||
fired = set()
|
||||
for t in all_tests:
|
||||
fired |= set(firing_assertions(t))
|
||||
missing = ALERTS - fired
|
||||
if missing:
|
||||
bad.append(f"no firing promtool assertion for {sorted(missing)}")
|
||||
for alert, metric in ABSENT_ALERTS.items():
|
||||
absent_case = any(
|
||||
alert in firing_assertions(t) and not any(s.startswith(metric) for s in series_for(t))
|
||||
for t in all_tests)
|
||||
if not absent_case:
|
||||
bad.append(f"{alert} has no firing test with NO {metric} series (the absent case)")
|
||||
vanish_case = any(
|
||||
alert in firing_assertions(t) and any(s.startswith(metric) and "stale" in str(i.get("values", ""))
|
||||
for i in t.get("input_series", []) for s in [i.get("series", "")])
|
||||
for t in all_tests)
|
||||
if not vanish_case:
|
||||
bad.append(f"{alert} has no test where its metric goes stale (the re-fire case)")
|
||||
return bad
|
||||
|
||||
|
||||
for v in eval_alerts(rules):
|
||||
check(False, f"(1,4) {v}")
|
||||
for v in eval_tests(tests):
|
||||
check(False, f"(2,3) {v}")
|
||||
check(len(alert_rules(rules)) == 5, "(1) five alert rules parsed")
|
||||
|
||||
# (5) the real harness, only when the rule engine is present.
|
||||
if shutil.which("promtool"):
|
||||
rc = subprocess.run(["bash", str(ROOT / "platform/monitoring/tests/run.sh"),
|
||||
"platform/vuln-scanner/alerts.promtool-test.yaml"],
|
||||
cwd=str(ROOT), capture_output=True, text=True)
|
||||
check(rc.returncode == 0, f"(5) the promtool harness failed: rc={rc.returncode}\n{rc.stdout[-800:]}{rc.stderr[-800:]}")
|
||||
|
||||
# (6) non-vacuity
|
||||
m = copy.deepcopy(rules)
|
||||
for g in m["spec"]["groups"]:
|
||||
for r in g["rules"]:
|
||||
if r["alert"] == "VulnTriageBotSilent":
|
||||
r["expr"] = r["expr"].split(" or ")[0]
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: absent arm deleted")
|
||||
m = copy.deepcopy(rules)
|
||||
m["spec"]["groups"][0]["rules"].pop(1)
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: alert dropped")
|
||||
m = copy.deepcopy(rules)
|
||||
for g in m["spec"]["groups"]:
|
||||
for r in g["rules"]:
|
||||
r["annotations"].pop("runbook_url", None)
|
||||
check(bool(eval_alerts(m)), "(6) evaluator missed: runbook_url removed")
|
||||
|
||||
m = copy.deepcopy(tests)
|
||||
for t in m["tests"]:
|
||||
t["promql_expr_test"] = [p for p in t["promql_expr_test"]
|
||||
if 'VulnCoverageGap' not in p.get("expr", "")]
|
||||
check(bool(eval_tests(m)), "(6) evaluator missed: firing assertion removed")
|
||||
m = copy.deepcopy(tests)
|
||||
m["tests"] = [t for t in m["tests"] if "ABSENT" not in t.get("name", "")]
|
||||
check(bool(eval_tests(m)), "(6) evaluator missed: absent-case tests removed")
|
||||
|
||||
if FAILURES:
|
||||
print(f"CHECK FAIL ({NAME}): {len(FAILURES)} of {CHECKS} assertions failed", file=sys.stderr)
|
||||
for f in FAILURES:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"PASS ({NAME}): {CHECKS} assertions")
|
||||
PY
|
||||
Reference in new issue
Block a user