home-network-k3s-impl/conformance/tests/l1-rendering/vuln-triage-bot-dashboard-aggregates/check.py

#!/usr/bin/env python3
"""L1: the CVE Dashboard issue is aggregates only (R-VULN-05, R-VULN-07).

Exercises the real renderer in .gitea/workflows/scripts/cve_triage.py over fixtures/report.json,
whose every name is distinctive so a leak cannot be confused with template text.

What this does NOT prove: that Gitea accepts the issue, or that the issue is the one the live bot
wrote. Those need a real run and are not exercised by any L1 test.
"""
import copy
import importlib.util
import json
import re
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path

sys.dont_write_bytecode = True
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[3]
spec = importlib.util.spec_from_file_location("cve_triage", ROOT / ".gitea/workflows/scripts/cve_triage.py")
bot = importlib.util.module_from_spec(spec)
sys.modules["cve_triage"] = bot
spec.loader.exec_module(bot)

RAW = (HERE / "fixtures/report.json").read_bytes()
FIXTURE = json.loads(RAW)
NOW = datetime(2026, 11, 9, 7, 15, tzinfo=timezone.utc)
PAGE = "https://cve.example.test"
failures = []


def check(name, condition):
    print(f"{'ok  ' if condition else 'FAIL'} {name}")
    if not condition:
        failures.append(name)


def sensitive_names(doc):
    """Every string in the fixture that must never reach the issue."""
    names = set()
    for target in doc["targets"]:
        for field in ("id", "image", "digest", "reason"):
            if target.get(field):
                names.add(target[field])
        if target.get("image"):
            names.add(bot.normalise_image(target["image"]))
            names.add(target["image"].rsplit("/", 1)[-1])
        for workload in target["workloads"]:
            names.update((workload["namespace"], workload["name"]))
            names.update(workload["containers"])
    for finding in doc["findings"]:
        for field in ("id", "package", "installedVersion", "fixedVersion", "title", "url"):
            if finding.get(field):
                names.add(finding[field])
        if finding.get("acceptance"):
            names.add(finding["acceptance"]["reason"])
    names.add(doc["database"]["version"])
    return {name for name in names if name}


def assert_no_names(body, names):
    lowered = body.lower()
    leaked = sorted(name for name in names if name.lower() in lowered)
    assert not leaked, f"{len(leaked)} fixture names leaked into the issue body"


def rendered(raw=RAW, now=NOW):
    return bot.render_dashboard(bot.aggregate(bot.parse_report(raw)), PAGE, now)


def fails(function):
    try:
        function()
    except AssertionError:
        return True
    return False


def rejected(doc):
    try:
        bot.parse_report(json.dumps(doc).encode())
    except bot.BotError:
        return True
    return False


body = rendered()
names = sensitive_names(FIXTURE)
check("fixture carries a non-trivial set of names", len(names) >= 25)
check("rendered body contains no fixture name", not fails(lambda: assert_no_names(body, names)))
check("body has no CVE or GHSA identifier at all", not re.search(r"CVE-\d|GHSA-", body, re.I))
check("hostile title's markup is absent", "onerror" not in body and "evil.example" not in body and "@keith" not in body)

# Non-vacuity: the assertion above must detect each class of leak.
for label, leak in {
    "CVE id": "CVE-2026-11111",
    "package": "fixture-libcrit",
    "image": "ghcr.io/fixture-org/fixture-api",
    "workload": "fixture-workload-db",
    "namespace": "fixture-ns-gamma",
}.items():
    check(f"non-vacuity: a leaked {label} is caught", fails(lambda: assert_no_names(body + leak, names)))

for row in (
    "| CRITICAL | 1 | 1 | 1 |",
    "| HIGH | 0 | 1 | 0 |",
    "| MEDIUM | 0 | 0 | 0 |",
    "| LOW | 1 | 0 | 0 |",
    "| UNKNOWN | 0 | 0 | 0 |",
    "| scanned | 1 |",
    "| pending | 0 |",
    "| stale | 1 |",
    "| failed | 0 |",
    "| credential-required | 1 |",
    "| by-tag | 0 |",
    "| not-covered | 1 |",
    "Unaccepted CRITICAL with a fix: **1**",
    "4 targets.",
    "| Report generated | 2026-11-09T06:00:00Z |",
    f"[CVE page]({PAGE})",
):
    check(f"body states {row!r}", row in body)
check("a fresh report carries no warning", "WARNING" not in body)

mutated = copy.deepcopy(FIXTURE)
extra = copy.deepcopy(mutated["findings"][0])
extra["id"] = "CVE-2026-55555"
mutated["findings"].append(extra)
changed = rendered(json.dumps(mutated).encode())
check("non-vacuity: a mutated fixture changes the counts", "| CRITICAL | 1 | 2 | 1 |" in changed and changed != body)

check("stale report warns", "older than 26 hours" in rendered(now=NOW + timedelta(hours=30)))
degraded = copy.deepcopy(FIXTURE)
degraded["run"]["status"] = "degraded"
check("degraded run warns", "did not complete cleanly" in rendered(json.dumps(degraded).encode()))
old_db = copy.deepcopy(FIXTURE)
old_db["database"]["updatedAt"] = "2026-11-01T00:00:00Z"
check("stale database warns", "database is older" in rendered(json.dumps(old_db).encode()))

for label, mutate in {
    "wrong schemaVersion": lambda d: d.update(schemaVersion=2),
    "unknown severity": lambda d: d["findings"][0].update(severity="SEVERE"),
    "unknown status": lambda d: d["findings"][0].update(status="ignored"),
    "unknown target state": lambda d: d["targets"][0].update(state="clean"),
    "finding on an unknown target": lambda d: d["findings"][0].update(targetId="nope"),
    "bad timestamp": lambda d: d.update(generatedAt="yesterday <b>"),
    "invalid image": lambda d: d["targets"][0].update(image="bad image <x>"),
}.items():
    doc = copy.deepcopy(FIXTURE)
    mutate(doc)
    check(f"parse rejects {label}", rejected(doc))

check("dashboard label is not one the PM acts on", bot.DASHBOARD_LABEL not in bot.FORBIDDEN_LABELS)
configmap = (ROOT / "platform/pm-agent/configmap.yaml").read_text()
match = re.search(r'^\s*PM_REQUIRE_LABEL:\s*"([^"]*)"', configmap, re.M)
check("PM_REQUIRE_LABEL is readable", match is not None)
check("the PM's label is in FORBIDDEN_LABELS and is not ours",
      match is not None and match.group(1) in bot.FORBIDDEN_LABELS and match.group(1) != bot.DASHBOARD_LABEL)

print(f"{len(failures)} failure(s)")
sys.exit(1 if failures else 0)