feat(vuln): scanner service and fixture-driven L1 tests
Stdlib-only scanner per docs/17: Kubernetes API inventory, Trivy runner, persisted state, schema-validated report.json, strict acceptances parser, server-rendered page, aggregate metrics, internal API, and a scan loop that stays live while scanning. L1 tests under conformance/tests/l1-rendering/vuln-scanner-* run the actual modules against recorded fixtures with a fake Trivy runner. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
4c3a284cad
commit
50ec2297ee
32 files changed
+3951
No files matched your search
@@ -0,0 +1,25 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-acceptances
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-08: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-08"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given acceptance files, When parsed and applied, Then expiry reverts to open and malformed entries never hide a finding.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-08: acceptance parsing, expiry and fail-safe behaviour."""
|
||||
import sys
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import acceptances # noqa: E402
|
||||
import report # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
TODAY = date(2026, 11, 9)
|
||||
NOW = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
def parse(text):
|
||||
return acceptances.parse_text(text, TODAY)
|
||||
|
||||
|
||||
entries, errors = parse("""# header comment
|
||||
- id: CVE-2026-1
|
||||
image: docker.io/library/nginx # exact match
|
||||
reason: not reachable - feature disabled
|
||||
expires: 2027-02-01
|
||||
|
||||
- id: CVE-2026-2
|
||||
reason: "platform wide"
|
||||
expires: 2026-12-01
|
||||
""")
|
||||
check("valid flat entries parse with no errors", len(entries) == 2 and errors == [])
|
||||
check("inline comment is stripped", entries[0]["image"] == "docker.io/library/nginx")
|
||||
check("omitted image means platform-wide", entries[1]["image"] is None)
|
||||
check("empty list form is accepted", parse("# none\n[]\n") == ([], []))
|
||||
check("empty file is accepted", parse("") == ([], []))
|
||||
|
||||
for name, text in (
|
||||
("missing reason", "- id: CVE-1\n expires: 2026-12-01\n"),
|
||||
("missing expires", "- id: CVE-1\n reason: x\n"),
|
||||
("expires beyond 180 days", "- id: CVE-1\n reason: x\n expires: 2027-06-01\n"),
|
||||
("unknown key", "- id: CVE-1\n reason: x\n expires: 2026-12-01\n scope: all\n"),
|
||||
("not a date", "- id: CVE-1\n reason: x\n expires: soon\n"),
|
||||
("impossible date", "- id: CVE-1\n reason: x\n expires: 2026-13-45\n"),
|
||||
("duplicate key", "- id: CVE-1\n reason: x\n reason: y\n expires: 2026-12-01\n"),
|
||||
("flow value", "- id: CVE-1\n reason: [x]\n expires: 2026-12-01\n"),
|
||||
):
|
||||
found, problems = parse(text)
|
||||
check("rejected: " + name, found == [] and len(problems) == 1)
|
||||
|
||||
found, problems = parse("- id: CVE-1\n reason: x\n expires: 2026-12-01\n- id: bad entry\n expires: 2026-12-01\n")
|
||||
check("one bad entry is ignored, the valid one kept", len(found) == 1 and len(problems) == 1)
|
||||
found, problems = parse("- id: CVE-1\n reason: x\n")
|
||||
check("bad indentation invalidates the whole file", found == [] and problems)
|
||||
found, problems = parse("garbage\n- id: CVE-1\n reason: x\n expires: 2026-12-01\n")
|
||||
check("a stray line invalidates the whole file", found == [] and problems)
|
||||
check("missing file is an error, not an empty allowlist", acceptances.load("/nonexistent/acceptances.yaml")[1])
|
||||
|
||||
entry = {"id": "CVE-1", "image": None, "reason": "r", "expires": date(2026, 11, 9)}
|
||||
check("acceptance holds on its expiry day", acceptances.match([entry], "CVE-1", "any/image", TODAY) is entry)
|
||||
check("acceptance gone the day after", acceptances.match([entry], "CVE-1", "any/image", TODAY + timedelta(days=1)) is None)
|
||||
scoped = {"id": "CVE-1", "image": "docker.io/library/nginx", "reason": "r", "expires": date(2026, 12, 1)}
|
||||
check("image match is exact", acceptances.match([scoped], "CVE-1", "docker.io/library/nginx-extra", TODAY) is None)
|
||||
check("image match accepts the exact image", acceptances.match([scoped], "CVE-1", "docker.io/library/nginx", TODAY) is scoped)
|
||||
|
||||
|
||||
def state_with(findings):
|
||||
target = {"id": "docker.io/library/nginx@" + A1, "type": "image", "image": "docker.io/library/nginx", "digest": A1,
|
||||
"ref": "", "reason": None, "workloads": []}
|
||||
return {
|
||||
"inventory": [target], "database": {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"},
|
||||
"run": {"startedAt": "2026-11-09T09:00:00Z", "finishedAt": "2026-11-09T09:30:00Z", "status": "complete"},
|
||||
"remediation": [], "inventoryError": None, "loadError": None,
|
||||
"scans": {A1: {"firstDiscoveredAt": "2026-11-01T00:00:00Z", "lastInventoriedAt": "2026-11-09T09:00:00Z",
|
||||
"lastAttemptAt": "2026-11-09T09:00:00Z", "lastSuccessAt": "2026-11-09T09:00:00Z", "outcome": "ok",
|
||||
"reason": None, "failedSince": None, "findings": findings,
|
||||
"firstSeen": {f["id"]: "2026-11-01T00:00:00Z" for f in findings}}},
|
||||
}
|
||||
|
||||
|
||||
FINDING = {"id": "CVE-1", "severity": "CRITICAL", "cvss": 9.0, "package": "p", "installedVersion": "1", "fixedVersion": "2",
|
||||
"title": "t", "url": ""}
|
||||
ST = state_with([FINDING])
|
||||
|
||||
|
||||
def status(when, accepted, errs=()):
|
||||
built = report.build(ST, accepted, list(errs), when)
|
||||
return built, built["findings"][0]
|
||||
|
||||
|
||||
entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)}
|
||||
built, finding = status(NOW, [entry])
|
||||
check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted"
|
||||
and finding["acceptance"] == {"reason": "not reachable", "expires": "2026-11-09"})
|
||||
check("accepted report is schema-valid", report.validate(built) == [])
|
||||
_, finding = status(NOW + timedelta(days=1), [entry])
|
||||
check("expired acceptance reverts to the finding's own status", finding["status"] == "fix-available" and "acceptance" not in finding)
|
||||
_, finding = status(NOW, [])
|
||||
check("mutation: no acceptance leaves it open/fix-available", finding["status"] == "fix-available")
|
||||
built, finding = status(NOW, [], ["bad entry"])
|
||||
check("an invalid entry degrades the run", built["run"]["status"] == "degraded")
|
||||
check("an invalid entry never hides the finding", finding["status"] == "fix-available")
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,28 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-api-metrics
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-02: "true"
|
||||
conformance.home.k3s/R-VULN-03: "true"
|
||||
conformance.home.k3s/R-VULN-05: "true"
|
||||
conformance.home.k3s/R-VULN-06: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-02, R-VULN-03, R-VULN-05, R-VULN-06"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given a running engine, When the HTTP surfaces are exercised, Then the API validates, metrics are aggregates only, and listeners answer during a scan.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-02/03/05/06: internal API, metrics, publish refusal, and HTTP during a scan."""
|
||||
import copy
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import main # noqa: E402
|
||||
import metrics # noqa: E402
|
||||
import report # noqa: E402
|
||||
import server # noqa: E402
|
||||
import state as st # noqa: E402
|
||||
|
||||
logging.disable(logging.CRITICAL)
|
||||
FAILED = []
|
||||
NOW = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = NOW
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
|
||||
class Client:
|
||||
def list_all(self, path):
|
||||
if path.endswith("pods"):
|
||||
return [{"metadata": {"namespace": "apps", "name": "p"}, "spec": {"containers": [{"name": "c", "image": "nginx:1"}]},
|
||||
"status": {"containerStatuses": [{"name": "c", "imageID": "docker.io/library/nginx@" + A1}]}}]
|
||||
return []
|
||||
|
||||
|
||||
class Runner:
|
||||
"""Blocks on release while a scan is 'running', so a test can prove the HTTP path stays live."""
|
||||
|
||||
def __init__(self):
|
||||
self.entered = threading.Event()
|
||||
self.release = threading.Event()
|
||||
self.release.set()
|
||||
self.result = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-1", "Severity": "CRITICAL", "PkgName": "openssl", "InstalledVersion": "1",
|
||||
"FixedVersion": "2", "Title": "t", "PrimaryURL": "https://example.test/1"}]}]}
|
||||
|
||||
def download_db(self):
|
||||
pass
|
||||
|
||||
def db_info(self):
|
||||
return {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"}
|
||||
|
||||
def scan(self, _reference):
|
||||
self.entered.set()
|
||||
self.release.wait(10)
|
||||
return self.result
|
||||
|
||||
def clean_cache(self):
|
||||
pass
|
||||
|
||||
|
||||
def request(port, method, path, body=None, headers=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
||||
conn.request(method, path, body, headers or {})
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
conn.close()
|
||||
return resp.status, data, resp
|
||||
|
||||
|
||||
def post(port, document, content_type="application/json"):
|
||||
return request(port, "POST", "/triage", json.dumps(document), {"Content-Type": content_type})
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
accept = os.path.join(tmp, "a.yaml")
|
||||
Path(accept).write_text("[]\n")
|
||||
clock = Clock()
|
||||
runner = Runner()
|
||||
eng = main.Engine(tmp, Client(), runner, accept, clock=clock, publish_seconds=0)
|
||||
servers = server.start_all(eng, ("127.0.0.1", 0), ("127.0.0.1", 0), ("127.0.0.1", 0))
|
||||
page_port, internal, metrics_port = (s.server_address[1] for s in servers)
|
||||
|
||||
status, body, _ = request(internal, "GET", "/report.json")
|
||||
check("report.json is 503 before any report", status == 503)
|
||||
status, body, resp = request(page_port, "GET", "/")
|
||||
check("page is 503 before any report, with the CSP", status == 503 and resp.getheader("Content-Security-Policy"))
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("last-complete-run metric is absent before the first run", "last_complete_run" not in text)
|
||||
check("triage heartbeat metric is absent before the first POST", "triage_heartbeat" not in text)
|
||||
|
||||
# HTTP must stay live while a scan is blocked mid-flight.
|
||||
runner.release.clear()
|
||||
worker = threading.Thread(target=eng.tick, daemon=True)
|
||||
worker.start()
|
||||
check("the scan is genuinely in flight", runner.entered.wait(10))
|
||||
status_h, _, _ = request(internal, "GET", "/healthz")
|
||||
status_p, _, _ = request(page_port, "GET", "/healthz")
|
||||
status_m, _, _ = request(metrics_port, "GET", "/metrics")
|
||||
check("listeners answer during a scan", (status_h, status_p, status_m) == (200, 200, 200))
|
||||
runner.release.set()
|
||||
worker.join(10)
|
||||
check("the tick then completes", not worker.is_alive())
|
||||
|
||||
status, body, _ = request(internal, "GET", "/report.json")
|
||||
check("report.json serves a schema-valid report after a tick", status == 200 and report.validate(json.loads(body)) == [])
|
||||
|
||||
# Page and CSP over HTTP
|
||||
status, body, resp = request(page_port, "GET", "/?view=inventory")
|
||||
check("page served over HTTP carries the CSP and nosniff",
|
||||
status == 200 and resp.getheader("Content-Security-Policy").startswith("default-src 'none'") and resp.getheader("X-Content-Type-Options") == "nosniff")
|
||||
status, body, resp = request(page_port, "GET", "/missing")
|
||||
check("a 404 from the page listener also carries the CSP", status == 404 and resp.getheader("Content-Security-Policy"))
|
||||
|
||||
status, body, resp = request(page_port, "POST", "/")
|
||||
check("an unsupported method on the page listener also carries the CSP", status == 501 and resp.getheader("Content-Security-Policy"))
|
||||
|
||||
# Triage API
|
||||
valid = {"heartbeatAt": "2026-11-09T11:00:00Z", "remediation": [
|
||||
{"image": "docker.io/library/nginx", "state": "pr-open", "url": "https://gitea.test/pulls/9", "updatedAt": "2026-11-09T11:00:00Z"}]}
|
||||
status, body, _ = post(internal, valid)
|
||||
check("valid triage is 204", status == 204 and body == b"")
|
||||
check("remediation surfaces in the report", eng.report["remediation"][0]["state"] == "pr-open")
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("heartbeat metric appears and uses server receipt time, not the bot's", f"triage_heartbeat_timestamp_seconds {int(NOW.timestamp())}" in text)
|
||||
for name, mutate in (
|
||||
("unknown top-level key", lambda d: d.update(extra=1)),
|
||||
("missing heartbeat", lambda d: d.pop("heartbeatAt")),
|
||||
("unknown remediation key", lambda d: d["remediation"][0].update(x=1)),
|
||||
("bad remediation state", lambda d: d["remediation"][0].update(state="merged")),
|
||||
("javascript url", lambda d: d["remediation"][0].update(url="javascript:x")),
|
||||
("bad timestamp", lambda d: d.update(heartbeatAt="yesterday")),
|
||||
):
|
||||
document = copy.deepcopy(valid)
|
||||
mutate(document)
|
||||
status, _, _ = post(internal, document)
|
||||
check("triage rejects " + name, status == 400)
|
||||
check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415)
|
||||
status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"})
|
||||
check("triage rejects an oversized body", status == 413)
|
||||
many = {"heartbeatAt": "2026-11-09T11:00:00Z", "remediation": [valid["remediation"][0]] * 5001}
|
||||
check("triage rejects more than 5000 entries", post(internal, many)[0] in (400, 413))
|
||||
check("rejected triage leaves state untouched", eng.state["remediation"][0]["url"] == "https://gitea.test/pulls/9")
|
||||
|
||||
# Metrics labels and values
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
allowed = {"state", "severity", "fixable"}
|
||||
labels = {k for line in text.splitlines() for k in re.findall(r"([A-Za-z_]+)=\"", line)}
|
||||
check("metric labels are within the allowlist", labels and labels <= allowed)
|
||||
check("no metric carries a CVE, package, image or workload", not re.search(r"CVE-|openssl|nginx|apps", text))
|
||||
for state in report.STATES:
|
||||
check("targets gauge present for " + state, 'vuln_scanner_targets{state="' + state + '"}' in text)
|
||||
check("critical fix findings counted with fixable true", 'vuln_scanner_findings{severity="CRITICAL",fixable="true"} 1' in text)
|
||||
check("every severity and fixability appears", text.count("vuln_scanner_findings{") == 10)
|
||||
check("failed-over-24h gauge exists", "vuln_scanner_failed_targets_over_24h 0" in text)
|
||||
check("critical fix overdue is 0 inside the 72h grace", "vuln_scanner_critical_fix_overdue 0" in text)
|
||||
clock.now += timedelta(hours=73)
|
||||
text = request(metrics_port, "GET", "/metrics")[1].decode()
|
||||
check("critical fix overdue is 1 after 72h", "vuln_scanner_critical_fix_overdue 1" in text)
|
||||
check("last-complete-run metric present after a complete run", "vuln_scanner_last_complete_run_timestamp_seconds" in text)
|
||||
for srv in servers:
|
||||
srv.shutdown()
|
||||
|
||||
# Invalid report: refuse and keep the old one.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
good = {"schemaVersion": 1, "generatedAt": "2026-11-09T10:00:00Z", "database": {"updatedAt": "2026-11-09T06:00:00Z", "version": "2"},
|
||||
"run": {"startedAt": "2026-11-09T09:00:00Z", "finishedAt": "2026-11-09T09:30:00Z", "status": "complete"},
|
||||
"targets": [], "findings": [], "remediation": []}
|
||||
check("a valid report publishes", report.publish(good, tmp) is True)
|
||||
before = Path(tmp, "report.json").read_bytes()
|
||||
for name, mutate in (
|
||||
("wrong schemaVersion", lambda r: r.update(schemaVersion=2)),
|
||||
("extra top-level key", lambda r: r.update(extra=1)),
|
||||
("bad timestamp", lambda r: r.update(generatedAt="soon")),
|
||||
("unknown state", lambda r: r.update(targets=[{"id": "x", "type": "image", "image": "x", "digest": None, "state": "fine",
|
||||
"reason": None, "lastScannedAt": None, "workloads": []}])),
|
||||
("dangling targetId", lambda r: r.update(findings=[{"id": "CVE-1", "targetId": "nope", "severity": "LOW", "cvss": None,
|
||||
"package": "p", "installedVersion": "1", "fixedVersion": None, "title": "t",
|
||||
"url": "", "firstSeenAt": "2026-11-01T00:00:00Z", "status": "open"}])),
|
||||
):
|
||||
broken = copy.deepcopy(good)
|
||||
mutate(broken)
|
||||
check("invalid report refused: " + name, report.publish(broken, tmp) is False)
|
||||
check("an invalid report writes nothing", Path(tmp, "report.json").read_bytes() == before)
|
||||
check("accepted without acceptance is invalid", report.validate({**good, "targets": [{"id": "x", "type": "image", "image": "x", "digest": None,
|
||||
"state": "scanned", "reason": None, "lastScannedAt": None, "workloads": []}], "findings": [{"id": "CVE-1", "targetId": "x", "severity": "LOW",
|
||||
"cvss": None, "package": "p", "installedVersion": "1", "fixedVersion": None, "title": "t", "url": "", "firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"status": "accepted"}]}) != [])
|
||||
Path(tmp, "report.json").write_text("{not json")
|
||||
check("a corrupt published report is not served", report.load_published(tmp) is None)
|
||||
|
||||
# Crash-consistency of state: the temp+rename leaves no temp files behind and the file parses.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
store = st.Store(tmp)
|
||||
state = st.new_state()
|
||||
state["triageHeartbeatAt"] = "2026-11-09T11:00:00Z"
|
||||
store.save(state)
|
||||
check("state saves with no temp file left behind", sorted(os.listdir(tmp)) == ["state.json"])
|
||||
check("state survives a reload", st.Store(tmp).load()["triageHeartbeatAt"] == "2026-11-09T11:00:00Z")
|
||||
Path(tmp, "state.json").write_text("not json")
|
||||
loaded = st.Store(tmp).load()
|
||||
check("corrupt state starts empty and marks degraded", loaded["loadError"] and loaded["scans"] == {})
|
||||
check("corrupt state is preserved for inspection", Path(tmp, "state.json.corrupt").exists())
|
||||
|
||||
# Metrics module directly, without the report.
|
||||
empty_state = st.new_state()
|
||||
text = metrics.render(None, empty_state, NOW)
|
||||
check("with no report only the state-derived gauge exists", text == "vuln_scanner_failed_targets_over_24h 0\n")
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,25 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-inventory
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-01: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-01"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given recorded API lists, When the inventory is built, Then owner chains, digests, by-tag CronJobs and not-covered targets resolve per R-VULN-01.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-01: inventory from recorded API lists."""
|
||||
import copy
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import inventory # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
CLUSTER = json.loads((HERE / "fixtures/cluster.json").read_text())
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
B2 = "sha256:" + "b2" * 32
|
||||
C3 = "sha256:" + "c3" * 32
|
||||
D4 = "sha256:" + "d4" * 32
|
||||
E5 = "sha256:" + "e5" * 32
|
||||
F6 = "sha256:" + "f6" * 32
|
||||
|
||||
|
||||
def by_id(targets):
|
||||
return {t["id"]: t for t in targets}
|
||||
|
||||
|
||||
def workloads(target):
|
||||
return {(w["namespace"], w["kind"], w["name"]): w["containers"] for w in target["workloads"]}
|
||||
|
||||
|
||||
targets = inventory.build(CLUSTER)
|
||||
ids = by_id(targets)
|
||||
|
||||
nginx = ids["docker.io/library/nginx@" + A1]
|
||||
check("ReplicaSet resolves to Deployment, not the ReplicaSet", workloads(nginx) == {
|
||||
("apps", "Deployment", "web"): ["app"], ("apps", "Deployment", "api"): ["server"]})
|
||||
check("docker-pullable:// prefix stripped and digest taken from imageID", nginx["digest"] == A1)
|
||||
check("init container inventoried", ("apps", "Deployment", "web") in workloads(ids["ghcr.io/org/tool@" + B2]))
|
||||
check("ephemeral container inventoried", "debug" in workloads(ids["docker.io/library/busybox@" + C3])[("apps", "Deployment", "web")])
|
||||
|
||||
backup = ids["ghcr.io/org/backup@" + E5]
|
||||
check("Job resolves to CronJob", workloads(backup) == {("ops", "CronJob", "backup"): ["backup"]})
|
||||
check("a CronJob with a running pod is not also inventoried from its template",
|
||||
not any(t["id"].startswith("ghcr.io/org/backup:") for t in targets))
|
||||
|
||||
nightly = ids["ghcr.io/org/nightly:2.1"]
|
||||
check("idle CronJob inventoried from template with no digest", nightly["digest"] is None
|
||||
and workloads(nightly) == {("ops", "CronJob", "nightly"): ["job"]})
|
||||
|
||||
solo = ids["docker.io/library/alpine:3.20"]
|
||||
check("empty imageID gives a by-tag target from the spec image", solo["digest"] is None)
|
||||
check("bare Pod is its own workload", ("ops", "Pod", "solo") in workloads(solo))
|
||||
check("Succeeded pod's image is not inventoried", not any(F6 in t["id"] for t in targets))
|
||||
check("virt-launcher stays owned by its VirtualMachineInstance",
|
||||
("kubevirt-vms", "VirtualMachineInstance", "vm1") in workloads(ids["quay.io/kubevirt/virt-launcher@sha256:" + "ab" * 32]))
|
||||
vm, node = ids["vm/kubevirt-vms/vm1"], ids["node/prod-1"]
|
||||
check("VM inventoried as non-container target", vm["type"] == "vm" and vm["digest"] is None and vm["reason"])
|
||||
check("node inventoried as non-container target", node["type"] == "node" and node["reason"])
|
||||
|
||||
# Non-vacuous: mutating the fixture changes the answer the assertions above rely on.
|
||||
broken = copy.deepcopy(CLUSTER)
|
||||
broken["replicasets"] = []
|
||||
broken["jobs"] = []
|
||||
broken_ids = by_id(inventory.build(broken))
|
||||
check("mutation: losing the ReplicaSet record leaves the ReplicaSet as the workload",
|
||||
("apps", "ReplicaSet", "web-5f") in workloads(broken_ids["docker.io/library/nginx@" + A1]))
|
||||
check("mutation: losing the Job record leaves the Job as the workload",
|
||||
("ops", "Job", "backup-123") in workloads(broken_ids["ghcr.io/org/backup@" + E5]))
|
||||
check("mutation: a Running phase on the Succeeded pod does inventory its image", any(
|
||||
F6 in t["id"] for t in inventory.build({**CLUSTER, "pods": [
|
||||
{**p, "status": {**p["status"], "phase": "Running"}} if p["metadata"]["name"] == "done-1" else p
|
||||
for p in CLUSTER["pods"]]})))
|
||||
|
||||
for reference, expected in (
|
||||
("nginx:1.27", ("docker.io/library/nginx", "1.27", None)),
|
||||
("nginx", ("docker.io/library/nginx", None, None)),
|
||||
("bitnami/redis:7", ("docker.io/bitnami/redis", "7", None)),
|
||||
("ghcr.io/org/app:v1", ("ghcr.io/org/app", "v1", None)),
|
||||
("registry.local:5000/x/y:2", ("registry.local:5000/x/y", "2", None)),
|
||||
("index.docker.io/library/nginx:1", ("docker.io/library/nginx", "1", None)),
|
||||
("ghcr.io/org/app:v1@" + A1, ("ghcr.io/org/app", "v1", A1)),
|
||||
):
|
||||
check("normalise " + reference, inventory.normalise_reference(reference) == expected)
|
||||
check("bare config id is not a digest", inventory.digest_from_image_id("sha256:" + "a1" * 32) is None)
|
||||
check("empty imageID has no digest", inventory.digest_from_image_id("") is None)
|
||||
|
||||
|
||||
class FakeListClient:
|
||||
"""list_all replaced; proves collect() tolerates exactly a 404 on the VMI API."""
|
||||
|
||||
def __init__(self, vmi_error):
|
||||
self.vmi_error = vmi_error
|
||||
|
||||
def list_all(self, path):
|
||||
if "kubevirt" in path and self.vmi_error:
|
||||
raise self.vmi_error
|
||||
return []
|
||||
|
||||
|
||||
check("VMI API 404 tolerated as no VMs", inventory.collect(FakeListClient(inventory.NotFound("x")))["vmis"] == [])
|
||||
try:
|
||||
inventory.collect(FakeListClient(inventory.InventoryError("403")))
|
||||
check("a non-404 VMI error is not swallowed", False)
|
||||
except inventory.InventoryError:
|
||||
check("a non-404 VMI error is not swallowed", True)
|
||||
|
||||
|
||||
class PagedAPI(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
if self.headers.get("Authorization") != "Bearer fixture-token":
|
||||
self.send_response(403)
|
||||
self.end_headers()
|
||||
return
|
||||
if self.path.startswith("/apis/kubevirt"):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
if "continue=" in self.path:
|
||||
body = {"items": [{"metadata": {"name": "b"}}], "metadata": {}}
|
||||
else:
|
||||
body = {"items": [{"metadata": {"name": "a"}}], "metadata": {"continue": "page2"}}
|
||||
payload = json.dumps(body).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
token = os.path.join(tmp, "token")
|
||||
Path(token).write_text("fixture-token\n")
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), PagedAPI)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
client = inventory.KubeClient("http://127.0.0.1:" + str(server.server_address[1]), token, os.devnull)
|
||||
items = client.list_all("/api/v1/pods")
|
||||
check("pagination follows the continue token with the bearer token", [i["metadata"]["name"] for i in items] == ["a", "b"])
|
||||
check("collect over real HTTP tolerates VMI 404", inventory.collect(client)["vmis"] == [])
|
||||
server.shutdown()
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,395 @@
|
||||
{
|
||||
"cronjobs": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"jobTemplate": {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/backup:1.0",
|
||||
"name": "job"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "nightly",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"jobTemplate": {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/nightly:2.1",
|
||||
"name": "job"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-123",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "CronJob",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-99",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "CronJob",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"nodes": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "prod-1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "prod-2"
|
||||
}
|
||||
}
|
||||
],
|
||||
"pods": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "web-5f"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"ephemeralContainers": [
|
||||
{
|
||||
"image": "busybox:1.37",
|
||||
"name": "debug"
|
||||
}
|
||||
],
|
||||
"initContainers": [
|
||||
{
|
||||
"image": "ghcr.io/org/tool:v2",
|
||||
"name": "init-db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker-pullable://docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"ephemeralContainerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/busybox@sha256:c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3",
|
||||
"name": "debug"
|
||||
}
|
||||
],
|
||||
"initContainerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/tool@sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2",
|
||||
"name": "init-db"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-2",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "web-5f"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"initContainers": [
|
||||
{
|
||||
"image": "ghcr.io/org/tool:v2",
|
||||
"name": "init-db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"initContainerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/tool@sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2",
|
||||
"name": "init-db"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "api-77"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "server"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "server"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "db-0",
|
||||
"namespace": "data",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "StatefulSet",
|
||||
"name": "db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "postgres:16",
|
||||
"name": "postgres"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/postgres@sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"name": "postgres"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "backup-123-x",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Job",
|
||||
"name": "backup-123"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/backup:1.0",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/backup@sha256:e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5",
|
||||
"name": "backup"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "solo",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "alpine:3.20",
|
||||
"name": "toolbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "",
|
||||
"name": "toolbox"
|
||||
}
|
||||
],
|
||||
"phase": "Pending"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "done-1",
|
||||
"namespace": "ops",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Job",
|
||||
"name": "backup-99"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/old-backup:0.1",
|
||||
"name": "backup"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/old-backup@sha256:f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6",
|
||||
"name": "backup"
|
||||
}
|
||||
],
|
||||
"phase": "Succeeded"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "virt-launcher-vm1-abc",
|
||||
"namespace": "kubevirt-vms",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "VirtualMachineInstance",
|
||||
"name": "vm1"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "quay.io/kubevirt/virt-launcher:v1.8",
|
||||
"name": "compute"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "quay.io/kubevirt/virt-launcher@sha256:abababababababababababababababababababababababababababababababab",
|
||||
"name": "compute"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
}
|
||||
],
|
||||
"replicasets": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-5f",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "web"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-77",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "api"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"vmis": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "vm1",
|
||||
"namespace": "kubevirt-vms"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-page
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-03: "true"
|
||||
conformance.home.k3s/R-VULN-04: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-03, R-VULN-04"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given a report fixture, When the page renders, Then all CVEs, containers and workloads appear in all views and hostile input is inert.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-03/04: the rendered page. Assertions run against the actual HTML output."""
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import page # noqa: E402
|
||||
|
||||
FAILED = []
|
||||
NOW = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
REPORT = json.loads((HERE / "fixtures/report.json").read_text())
|
||||
CLEAN_RE = re.compile(r"(?i)\bclean\b|no vulnerabilit")
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
def render(report=REPORT, query="", now=NOW):
|
||||
status, headers, body = page.render(report, query, now)
|
||||
return status, headers, body.decode("utf-8")
|
||||
|
||||
|
||||
class Tags(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.tags = []
|
||||
self.attrs = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
self.tags.append(tag)
|
||||
self.attrs.append((tag, dict(attrs)))
|
||||
|
||||
|
||||
def tags(body):
|
||||
parser = Tags()
|
||||
parser.feed(body)
|
||||
return parser
|
||||
|
||||
|
||||
def covers_everything(body, report):
|
||||
"""Every fixture CVE id, workload name and container name appears in the document."""
|
||||
missing = [f["id"] for f in report["findings"] if f["id"] not in body]
|
||||
for target in report["targets"]:
|
||||
for workload in target["workloads"]:
|
||||
if workload["name"] not in body:
|
||||
missing.append(workload["name"])
|
||||
missing.extend(c for c in workload["containers"] if c not in body)
|
||||
return missing
|
||||
|
||||
|
||||
def workload_view_covers(body, report):
|
||||
"""Every workload row with its containers, and every CVE in the expansions."""
|
||||
missing = covers_everything(body, report)
|
||||
return missing
|
||||
|
||||
|
||||
for view in ("cves", "workloads", "inventory"):
|
||||
status, headers, body = render(query="view=" + view)
|
||||
check(view + " view returns 200", status == 200)
|
||||
check(view + " view has the Content-Security-Policy with no script source", "script-src" not in headers["Content-Security-Policy"]
|
||||
and headers["Content-Security-Policy"].startswith("default-src 'none'"))
|
||||
check(view + " view sets nosniff", headers["X-Content-Type-Options"] == "nosniff")
|
||||
check(view + " view has no script element", "script" not in tags(body).tags)
|
||||
style_body = re.search(r"<style>(.*?)</style>", body, re.S).group(1)
|
||||
expected = base64.b64encode(hashlib.sha256(style_body.encode()).digest()).decode()
|
||||
check(view + " view's CSP hash matches its inline style", "'sha256-" + expected + "'" in headers["Content-Security-Policy"])
|
||||
check(view + " view header shows every state count", all(state + ": <strong>" in body for state in page.report_mod.STATES))
|
||||
check(view + " view shows a coverage warning", "INCOMPLETE COVERAGE" in body)
|
||||
check(view + " view makes no clean claim with unscanned targets", not CLEAN_RE.search(body))
|
||||
|
||||
status, headers, cves = render(query="view=cves")
|
||||
check("CVE view contains every CVE", all(f["id"] in cves for f in REPORT["findings"]))
|
||||
affected = {f["targetId"] for f in REPORT["findings"]}
|
||||
expected_in_cves = {(t["id"], w["name"]) for t in REPORT["targets"] if t["id"] in affected for w in t["workloads"]}
|
||||
check("CVE view expansions name every workload of an affected target", all(name in cves for _, name in expected_in_cves))
|
||||
check("CVE view expansions name every container of an affected target",
|
||||
all(c in cves for t in REPORT["targets"] if t["id"] in affected for w in t["workloads"] for c in w["containers"]))
|
||||
for name in ("web", "api", "app", "server"):
|
||||
check("CVE view expansion names " + name, name in cves)
|
||||
status, headers, workloads = render(query="view=workloads")
|
||||
check("workloads view contains every workload and container, including zero-finding ones", covers_everything(workloads, REPORT) == [])
|
||||
status, headers, inventory = render(query="view=inventory")
|
||||
check("inventory view contains every target", all(t["id"] in inventory for t in REPORT["targets"]))
|
||||
check("inventory view contains every workload and container", covers_everything(inventory, REPORT) == [])
|
||||
mutated = copy.deepcopy(REPORT)
|
||||
mutated["findings"] = [f for f in mutated["findings"] if f["id"] != "CVE-2026-1003"]
|
||||
check("mutation: the coverage evaluator catches a dropped CVE", "CVE-2026-1003" in covers_everything(render(mutated, "view=workloads")[2], REPORT))
|
||||
|
||||
_, _, body = render(query="view=cves")
|
||||
check("hostile title is escaped", "<script>alert(1)</script>" in body)
|
||||
check("hostile title is never live", "<script>" not in body)
|
||||
hostile = copy.deepcopy(REPORT)
|
||||
hostile["findings"][0]["url"] = "javascript:alert(1)"
|
||||
hostile["findings"][0]["title"] = '"><img src=x onerror=alert(1)>'
|
||||
hostile["targets"][0]["reason"] = "<b onmouseover=alert(1)>x</b>"
|
||||
hostile["remediation"][0]["url"] = "javascript:evil()"
|
||||
_, _, body = render(hostile, "view=cves")
|
||||
parsed = tags(body)
|
||||
check("a javascript: advisory URL never becomes a link", not any("javascript:" in (a.get("href") or "") for t, a in parsed.attrs if t == "a"))
|
||||
check("no element carries a hostile inline handler", not any(k.startswith("on") for _, a in parsed.attrs for k in a))
|
||||
check("no img element was injected", "img" not in parsed.tags)
|
||||
_, _, body = render(hostile, "view=inventory")
|
||||
check("hostile reason is escaped in inventory", "<b onmouseover" not in body and "<b onmouseover" in body)
|
||||
_, _, body = render(query="view=cves&q=" + "%3Cscript%3E")
|
||||
check("a hostile search term is escaped", "<script>" not in body)
|
||||
check("a linked CVE points at its advisory", 'href="https://example.test/CVE-2026-1001"' in render(query="view=cves")[2])
|
||||
|
||||
stale = copy.deepcopy(REPORT)
|
||||
stale["generatedAt"] = (NOW - timedelta(hours=27)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
_, _, body = render(stale)
|
||||
check("stale report shows a warning", "STALE REPORT" in body)
|
||||
check("stale report shows no normal state", "Status: ATTENTION NEEDED" in body and "Status: CURRENT" not in body)
|
||||
check("mutation: a fresh report has no stale warning", "STALE REPORT" not in render(REPORT)[2])
|
||||
degraded = copy.deepcopy(REPORT)
|
||||
degraded["run"]["status"] = "degraded"
|
||||
_, _, body = render(degraded)
|
||||
check("degraded run shows a warning and no normal state", "DEGRADED RUN" in body and "Status: ATTENTION NEEDED" in body and "Status: CURRENT" not in body)
|
||||
|
||||
allgood = copy.deepcopy(REPORT)
|
||||
allgood["targets"] = [t for t in allgood["targets"] if t["state"] == "scanned"]
|
||||
allgood["findings"] = []
|
||||
_, _, body = render(allgood)
|
||||
check("the clean claim appears when every target is scanned with zero findings", "Clean: all 1 targets" in body)
|
||||
check("mutation: an all-scanned report shows the current state", "Status: CURRENT" in body)
|
||||
one_pending = copy.deepcopy(allgood)
|
||||
one_pending["targets"].append({"id": "x@sha256:" + "0" * 64, "type": "image", "image": "x", "digest": "sha256:" + "0" * 64,
|
||||
"state": "pending", "reason": None, "lastScannedAt": None, "workloads": []})
|
||||
_, _, body = render(one_pending)
|
||||
check("one pending target removes the clean claim", "Clean:" not in body and not CLEAN_RE.search(body))
|
||||
for state in ("failed", "stale", "credential-required", "by-tag", "not-covered"):
|
||||
variant = copy.deepcopy(allgood)
|
||||
variant["targets"].append({**one_pending["targets"][-1], "state": state})
|
||||
check("a " + state + " target removes the clean claim", not CLEAN_RE.search(render(variant)[2]))
|
||||
empty = copy.deepcopy(allgood)
|
||||
empty["targets"] = []
|
||||
check("zero targets is never a clean claim", not CLEAN_RE.search(render(empty)[2]))
|
||||
degraded_clean = copy.deepcopy(allgood)
|
||||
degraded_clean["run"]["status"] = "degraded"
|
||||
check("a degraded run never makes a clean claim", not CLEAN_RE.search(render(degraded_clean)[2]))
|
||||
|
||||
status, headers, body = render(None)
|
||||
check("no report yet is a 503 with the CSP", status == 503 and "Content-Security-Policy" in headers)
|
||||
check("no report yet makes no clean claim", not CLEAN_RE.search(body))
|
||||
|
||||
_, _, body = render(query="view=cves&severity=CRITICAL")
|
||||
check("the severity filter narrows the list", "CVE-2026-1001" in body and "CVE-2026-1004" not in body)
|
||||
check("a filter is carried in pagination and sort links", "severity=CRITICAL" in body)
|
||||
_, _, body = render(query="view=cves&status=accepted")
|
||||
check("accepted findings stay listed with their reason", "CVE-2026-1003" in body and "not reachable" in body)
|
||||
_, _, body = render(query="view=cves&fix=no")
|
||||
check("the fix filter excludes fixed CVEs", "CVE-2026-1001" not in body and "CVE-2026-1002" in body)
|
||||
_, _, body = render(query="view=inventory&status=credential-required")
|
||||
check("inventory filters on coverage state", "ghcr.io/org/private" in body and "postgres" not in body.split("<main>")[1])
|
||||
_, _, body = render(query="view=workloads&namespace=ops")
|
||||
check("the namespace filter narrows workloads", "nightly" in body and "<td>web</td>" not in body)
|
||||
_, _, body = render(query="view=cves&q=openssl")
|
||||
check("free text search matches packages", "CVE-2026-1001" in body and "CVE-2026-1004" not in body)
|
||||
_, _, body = render()
|
||||
check("default view excludes nothing", all(f["id"] in body for f in REPORT["findings"]))
|
||||
check("remediation column shows pr-open with a link", 'href="https://gitea.test/pulls/1"' in body and "pr-open" in body)
|
||||
|
||||
big = copy.deepcopy(REPORT)
|
||||
big["findings"] = [dict(REPORT["findings"][0], id="CVE-2026-" + format(i, "05d")) for i in range(250)]
|
||||
_, _, body = render(big, "view=cves&size=50&page=2")
|
||||
check("pagination states the total", "of 250 CVEs (page 2 of 5)" in body)
|
||||
check("pagination shows only its page", "CVE-2026-00049" not in body and "CVE-2026-00050" in body and "CVE-2026-00100" not in body)
|
||||
check("every page's links include next and previous", ">next<" in body and ">previous<" in body)
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,199 @@
|
||||
{
|
||||
"database": {
|
||||
"updatedAt": "2026-11-09T06:00:00Z",
|
||||
"version": "2"
|
||||
},
|
||||
"findings": [
|
||||
{
|
||||
"cvss": 9.1,
|
||||
"firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"fixedVersion": "3.0.2",
|
||||
"id": "CVE-2026-1001",
|
||||
"installedVersion": "1.0",
|
||||
"package": "openssl",
|
||||
"severity": "CRITICAL",
|
||||
"status": "fix-available",
|
||||
"targetId": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"title": "OpenSSL flaw",
|
||||
"url": "https://example.test/CVE-2026-1001"
|
||||
},
|
||||
{
|
||||
"cvss": null,
|
||||
"firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"fixedVersion": null,
|
||||
"id": "CVE-2026-1002",
|
||||
"installedVersion": "1.0",
|
||||
"package": "zlib",
|
||||
"severity": "LOW",
|
||||
"status": "open",
|
||||
"targetId": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"title": "<script>alert(1)</script>",
|
||||
"url": "https://example.test/CVE-2026-1002"
|
||||
},
|
||||
{
|
||||
"acceptance": {
|
||||
"expires": "2027-01-01",
|
||||
"reason": "not reachable"
|
||||
},
|
||||
"cvss": null,
|
||||
"firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"fixedVersion": "16.2",
|
||||
"id": "CVE-2026-1003",
|
||||
"installedVersion": "1.0",
|
||||
"package": "libpq",
|
||||
"severity": "HIGH",
|
||||
"status": "accepted",
|
||||
"targetId": "docker.io/library/postgres@sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"title": "libpq flaw (retained from a prior scan)",
|
||||
"url": "https://example.test/CVE-2026-1003"
|
||||
},
|
||||
{
|
||||
"cvss": null,
|
||||
"firstSeenAt": "2026-11-01T00:00:00Z",
|
||||
"fixedVersion": null,
|
||||
"id": "CVE-2026-1004",
|
||||
"installedVersion": "1.0",
|
||||
"package": "curl",
|
||||
"severity": "MEDIUM",
|
||||
"status": "open",
|
||||
"targetId": "ghcr.io/org/nightly:2.1",
|
||||
"title": "curl flaw",
|
||||
"url": "https://example.test/CVE-2026-1004"
|
||||
}
|
||||
],
|
||||
"generatedAt": "2026-11-09T10:00:00Z",
|
||||
"remediation": [
|
||||
{
|
||||
"image": "docker.io/library/nginx",
|
||||
"state": "pr-open",
|
||||
"updatedAt": "2026-11-09T10:00:00Z",
|
||||
"url": "https://gitea.test/pulls/1"
|
||||
}
|
||||
],
|
||||
"run": {
|
||||
"finishedAt": "2026-11-09T09:30:00Z",
|
||||
"startedAt": "2026-11-09T09:00:00Z",
|
||||
"status": "complete"
|
||||
},
|
||||
"schemaVersion": 1,
|
||||
"targets": [
|
||||
{
|
||||
"digest": "sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"id": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"image": "docker.io/library/nginx",
|
||||
"lastScannedAt": "2026-11-09T10:00:00Z",
|
||||
"reason": null,
|
||||
"state": "scanned",
|
||||
"type": "image",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [
|
||||
"app"
|
||||
],
|
||||
"kind": "Deployment",
|
||||
"name": "web",
|
||||
"namespace": "apps"
|
||||
},
|
||||
{
|
||||
"containers": [
|
||||
"server"
|
||||
],
|
||||
"kind": "Deployment",
|
||||
"name": "api",
|
||||
"namespace": "apps"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"digest": "sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"id": "docker.io/library/postgres@sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"image": "docker.io/library/postgres",
|
||||
"lastScannedAt": "2026-11-07T09:00:00Z",
|
||||
"reason": "trivy timed out",
|
||||
"state": "failed",
|
||||
"type": "image",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [
|
||||
"postgres"
|
||||
],
|
||||
"kind": "StatefulSet",
|
||||
"name": "db",
|
||||
"namespace": "data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"digest": "sha256:e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5",
|
||||
"id": "ghcr.io/org/private@sha256:e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5",
|
||||
"image": "ghcr.io/org/private",
|
||||
"lastScannedAt": null,
|
||||
"reason": "registry ghcr.io refused anonymous access",
|
||||
"state": "credential-required",
|
||||
"type": "image",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [
|
||||
"priv"
|
||||
],
|
||||
"kind": "Deployment",
|
||||
"name": "priv",
|
||||
"namespace": "apps"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"digest": null,
|
||||
"id": "ghcr.io/org/nightly:2.1",
|
||||
"image": "ghcr.io/org/nightly",
|
||||
"lastScannedAt": "2026-11-09T10:00:00Z",
|
||||
"reason": null,
|
||||
"state": "by-tag",
|
||||
"type": "image",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [
|
||||
"job"
|
||||
],
|
||||
"kind": "CronJob",
|
||||
"name": "nightly",
|
||||
"namespace": "ops"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"digest": null,
|
||||
"id": "vm/kubevirt-vms/vm1",
|
||||
"image": "",
|
||||
"lastScannedAt": null,
|
||||
"reason": "guest OS packages are not assessed by this scanner",
|
||||
"state": "not-covered",
|
||||
"type": "vm",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [],
|
||||
"kind": "VirtualMachineInstance",
|
||||
"name": "vm1",
|
||||
"namespace": "kubevirt-vms"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"digest": null,
|
||||
"id": "node/prod-1",
|
||||
"image": "",
|
||||
"lastScannedAt": null,
|
||||
"reason": "node OS packages and the k3s binary are not assessed by this scanner",
|
||||
"state": "not-covered",
|
||||
"type": "node",
|
||||
"workloads": [
|
||||
{
|
||||
"containers": [],
|
||||
"kind": "Node",
|
||||
"name": "prod-1",
|
||||
"namespace": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# L1 fixture test: runs check.py (python3 stdlib, source imported from platform/vuln-scanner/src,
|
||||
# no cluster, no network beyond loopback). conformance/run.sh --layer l1 runs check.sh offline;
|
||||
# this Chainsaw Test wraps the same script so an in-cluster run executes identical assertions.
|
||||
apiVersion: chainsaw.kyverno.io/v1alpha1
|
||||
kind: Test
|
||||
metadata:
|
||||
name: vuln-scanner-scan-loop
|
||||
labels:
|
||||
conformance.home.k3s/R-VULN-02: "true"
|
||||
conformance.home.k3s/R-VULN-03: "true"
|
||||
conformance.home.k3s/R-VULN-05: "true"
|
||||
annotations:
|
||||
conformance.home.k3s/requirements: "R-VULN-02, R-VULN-03, R-VULN-05"
|
||||
conformance.home.k3s/layer: "L1"
|
||||
conformance.home.k3s/safety-critical: "false"
|
||||
# Whole-repo scope: the touched-path map keys nothing to platform/vuln-scanner, so without this
|
||||
# a change to the scanner would select no vuln-scanner test.
|
||||
conformance.home.k3s/scope: "whole-repo"
|
||||
conformance.home.k3s/intent: >-
|
||||
Given a fake Trivy runner and clock, When ticks run, Then shared digests scan once, failures isolate, and coverage states and first-seen persist.
|
||||
spec:
|
||||
steps:
|
||||
- name: run-offline-check
|
||||
description: Run check.sh, the same assertions the offline harness runs.
|
||||
try:
|
||||
- script:
|
||||
content: bash ./check.sh
|
||||
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R-VULN-02/03/05: scan loop semantics with a fake Trivy runner and a controllable clock."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[3] / "platform/vuln-scanner/src"))
|
||||
import main # noqa: E402
|
||||
import report # noqa: E402
|
||||
import trivy # noqa: E402
|
||||
|
||||
logging.disable(logging.CRITICAL)
|
||||
FAILED = []
|
||||
A1 = "sha256:" + "a1" * 32
|
||||
D4 = "sha256:" + "d4" * 32
|
||||
E5 = "sha256:" + "e5" * 32
|
||||
NGINX = "docker.io/library/nginx@" + A1
|
||||
POSTGRES = "docker.io/library/postgres@" + D4
|
||||
PRIVATE = "ghcr.io/org/private@" + E5
|
||||
NIGHTLY = "ghcr.io/org/nightly:2.1"
|
||||
CLUSTER = json.loads((HERE / "fixtures/cluster.json").read_text())
|
||||
|
||||
|
||||
def fixture(name):
|
||||
return json.loads((HERE / "fixtures" / name).read_text())
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
if not condition:
|
||||
FAILED.append(name)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = datetime(2026, 11, 9, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
def advance(self, **delta):
|
||||
self.now += timedelta(**delta)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, cluster):
|
||||
self.cluster = cluster
|
||||
self.fail = False
|
||||
|
||||
def list_all(self, path):
|
||||
if self.fail:
|
||||
raise RuntimeError("api down")
|
||||
for name, item in (("pods", "/api/v1/pods"), ("replicasets", "replicasets"), ("jobs", "/jobs"),
|
||||
("cronjobs", "cronjobs"), ("nodes", "/nodes"), ("vmis", "virtualmachineinstances")):
|
||||
if path.endswith(item):
|
||||
return self.cluster[name]
|
||||
return []
|
||||
|
||||
|
||||
class FakeRunner:
|
||||
"""Recorded results, no Trivy binary."""
|
||||
|
||||
def __init__(self):
|
||||
self.scans = []
|
||||
self.outcomes = {
|
||||
NGINX: fixture("trivy-nginx.json"), POSTGRES: fixture("trivy-postgres.json"),
|
||||
NIGHTLY: fixture("trivy-nightly.json"),
|
||||
PRIVATE: trivy.classify_failure("ghcr.io", "FATAL: GET https://ghcr.io/: UNAUTHORIZED: authentication required"),
|
||||
}
|
||||
self.db_age_hours = 1
|
||||
self.db_fails = False
|
||||
self.clock = None
|
||||
|
||||
def download_db(self):
|
||||
if self.db_fails:
|
||||
raise trivy.ScanFailure("failed", "database update failed")
|
||||
|
||||
def db_info(self):
|
||||
return {"updatedAt": (self.clock() - timedelta(hours=self.db_age_hours)).strftime("%Y-%m-%dT%H:%M:%SZ"), "version": "2"}
|
||||
|
||||
def scan(self, reference):
|
||||
self.scans.append(reference)
|
||||
outcome = self.outcomes[reference]
|
||||
if isinstance(outcome, Exception):
|
||||
raise outcome
|
||||
return outcome
|
||||
|
||||
def clean_cache(self):
|
||||
pass
|
||||
|
||||
|
||||
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER):
|
||||
path = os.path.join(tmp, "acceptances.yaml")
|
||||
if not os.path.exists(path):
|
||||
Path(path).write_text(acc)
|
||||
clock = clock or Clock()
|
||||
runner = runner or FakeRunner()
|
||||
runner.clock = clock
|
||||
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock
|
||||
|
||||
|
||||
def targets(eng):
|
||||
return {t["id"]: t for t in eng.report["targets"]}
|
||||
|
||||
|
||||
def findings(eng, target_id):
|
||||
return [f for f in eng.report["findings"] if f["targetId"] == target_id]
|
||||
|
||||
|
||||
def one_scan_per_digest(scans):
|
||||
return len(scans) == len(set(scans))
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
t = targets(eng)
|
||||
check("two workloads sharing a digest produce exactly one scan", runner.scans.count(NGINX) == 1 and one_scan_per_digest(runner.scans))
|
||||
check("mutation: the digest-uniqueness evaluator rejects a double scan", not one_scan_per_digest(["a", "a"]))
|
||||
check("the shared digest maps back to both workloads",
|
||||
{(w["kind"], w["name"]) for w in t[NGINX]["workloads"]} == {("Deployment", "web"), ("Deployment", "api")})
|
||||
check("shared digest findings appear once per target", len(findings(eng, NGINX)) == 2)
|
||||
check("digest target is scanned", t[NGINX]["state"] == "scanned")
|
||||
check("idle CronJob template is by-tag after scanning, not scanned", t[NIGHTLY]["state"] == "by-tag")
|
||||
check("node is not-covered", t["node/prod-1"]["state"] == "not-covered")
|
||||
check("report is schema-valid", report.validate(eng.report) == [])
|
||||
check("credential-required is its own state, not failed", t[PRIVATE]["state"] == "credential-required")
|
||||
check("credential-required reason names the registry", "ghcr.io" in t[PRIVATE]["reason"])
|
||||
check("credential-required does not count toward failed_over_24h", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0)
|
||||
check("report is written to disk", json.loads(Path(tmp, "report.json").read_text())["generatedAt"])
|
||||
first_seen = {f["id"]: f["firstSeenAt"] for f in findings(eng, NGINX)}
|
||||
|
||||
clock.advance(hours=21)
|
||||
runner.outcomes[POSTGRES] = trivy.ScanFailure("failed", "pull error: manifest unknown")
|
||||
eng.tick()
|
||||
t = targets(eng)
|
||||
check("a failed image leaves the run continuing", runner.scans.count(NGINX) == 2 and runner.scans.count(NIGHTLY) == 2)
|
||||
check("the failed target is failed", t[POSTGRES]["state"] == "failed")
|
||||
check("the failed target's reason is shown", "manifest unknown" in t[POSTGRES]["reason"])
|
||||
check("the failed target keeps its prior findings", [f["id"] for f in findings(eng, POSTGRES)] == ["CVE-2026-0010"])
|
||||
check("other targets stay scanned", t[NGINX]["state"] == "scanned")
|
||||
check("first-seen survives a re-scan", {f["id"]: f["firstSeenAt"] for f in findings(eng, NGINX)} == first_seen)
|
||||
check("run with one failed image is still complete (failure is per-target)", eng.report["run"]["status"] == "complete")
|
||||
|
||||
clock.advance(hours=25)
|
||||
check("failed target is counted over 24h after 25h", report.failed_over_24h(eng.state, clock()) == 1)
|
||||
eng.publish()
|
||||
check("a target whose last success aged past 24h is stale", targets(eng)[NGINX]["state"] == "stale")
|
||||
check("stale target keeps findings", len(findings(eng, NGINX)) == 2)
|
||||
|
||||
# Restart: a new Engine loads everything from disk.
|
||||
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
|
||||
check("report survives restart", eng2.report is not None and eng2.report_bytes() is not None)
|
||||
check("last complete run survives restart", eng2.state["lastCompleteRunAt"] is not None)
|
||||
eng2.publish()
|
||||
check("first-seen survives restart", {f["id"]: f["firstSeenAt"] for f in findings(eng2, NGINX)} == first_seen)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Not-scanned is never zero findings: the first tick has not scanned anything yet.
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.refresh_inventory()
|
||||
eng.publish()
|
||||
t = targets(eng)
|
||||
check("before scanning, digest targets are pending", t[NGINX]["state"] == "pending" and t[POSTGRES]["state"] == "pending")
|
||||
check("run object exists and is degraded before the first run", eng.report["run"]["status"] == "degraded")
|
||||
check("no fabricated findings before scanning", eng.report["findings"] == [])
|
||||
check("zero findings report still counts every target", len(eng.report["targets"]) == 5)
|
||||
clock.advance(hours=3)
|
||||
eng.publish()
|
||||
t = targets(eng)
|
||||
check("pending beyond 2h becomes failed, never silently absent", t[NGINX]["state"] == "failed")
|
||||
check("failed reason is the contract's text", t[NGINX]["reason"] == report.NOT_SCANNED_REASON)
|
||||
check("a never-scanned failed target has lastScannedAt null", t[NGINX]["lastScannedAt"] is None)
|
||||
clock.advance(hours=24)
|
||||
check("never-scanned target counts as failed over 24h", report.failed_over_24h(eng.state, clock()) >= 1)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Database problems degrade the run but it still reports.
|
||||
eng, runner, clock = engine(tmp)
|
||||
runner.db_fails = True
|
||||
eng.tick()
|
||||
check("DB update failure degrades the run", eng.report["run"]["status"] == "degraded")
|
||||
check("DB update failure still scans and reports", runner.scans and eng.report["findings"])
|
||||
check("a degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
runner.db_age_hours = 49
|
||||
eng.tick()
|
||||
check("a DB older than 48h degrades the run", eng.report["run"]["status"] == "degraded")
|
||||
runner.db_age_hours = 1
|
||||
clock.advance(hours=21)
|
||||
eng.tick()
|
||||
check("mutation: a fresh DB gives a complete run", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A target leaving the inventory leaves the report, with its findings.
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
check("postgres is present first", POSTGRES in targets(eng))
|
||||
gone = dict(CLUSTER)
|
||||
gone["pods"] = [p for p in CLUSTER["pods"] if p["metadata"]["name"] != "db-0"]
|
||||
eng.client.cluster = gone
|
||||
clock.advance(minutes=15)
|
||||
eng.tick()
|
||||
check("a target leaving the inventory leaves the report", POSTGRES not in targets(eng))
|
||||
check("its findings go with it", not findings(eng, POSTGRES))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Unexpected exceptions are contained at each level.
|
||||
eng, runner, clock = engine(tmp)
|
||||
runner.outcomes[POSTGRES] = RuntimeError("boom")
|
||||
eng.tick()
|
||||
check("an unexpected scan exception fails only that target", targets(eng)[POSTGRES]["state"] == "failed")
|
||||
check("the run continued past it", runner.scans.count(NIGHTLY) == 1)
|
||||
eng.client.fail = True
|
||||
clock.advance(minutes=15)
|
||||
eng.tick()
|
||||
check("an inventory failure keeps the previous inventory", NGINX in targets(eng))
|
||||
check("an inventory failure degrades the report", eng.report["run"]["status"] == "degraded")
|
||||
|
||||
calls = []
|
||||
|
||||
def crashing_tick():
|
||||
calls.append(1)
|
||||
if len(calls) == 1:
|
||||
raise RuntimeError("cycle crash")
|
||||
eng.stop.set()
|
||||
|
||||
eng.tick = crashing_tick
|
||||
eng.tick_seconds = 0
|
||||
thread = threading.Thread(target=eng.run_forever, daemon=True)
|
||||
thread.start()
|
||||
thread.join(10)
|
||||
check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive())
|
||||
|
||||
# Recorded Trivy output through the normaliser: untrusted text.
|
||||
parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y",
|
||||
"PrimaryURL": "javascript:alert(1)", "CVSS": {"nvd": {"V3Score": 11}}},
|
||||
{"VulnerabilityID": "CVE-2", "Severity": "HIGH", "PkgName": "q", "InstalledVersion": "1", "FixedVersion": "2",
|
||||
"PrimaryURL": "https://example.test/x", "CVSS": {"nvd": {"V3Score": 7.5}}},
|
||||
]}]})
|
||||
check("unknown severity becomes UNKNOWN", parsed[0]["severity"] == "UNKNOWN")
|
||||
check("control characters are stripped from upstream text", "\x07" not in parsed[0]["title"])
|
||||
check("non-http(s) advisory URL becomes empty", parsed[0]["url"] == "")
|
||||
check("out-of-range CVSS is dropped", parsed[0]["cvss"] is None)
|
||||
check("http(s) URL is kept", parsed[1]["url"] == "https://example.test/x")
|
||||
check("a recorded report with no Results is zero findings", trivy.parse_results(fixture("trivy-empty.json")) == [])
|
||||
try:
|
||||
trivy.parse_results({"not": "a report"})
|
||||
check("an unrecognised document is a failure, never zero findings", False)
|
||||
except trivy.ScanFailure as exc:
|
||||
check("an unrecognised document is a failure, never zero findings", exc.kind == "failed")
|
||||
failure = trivy.classify_failure("ghcr.io", "GET failed: 401 Unauthorized")
|
||||
check("401 is credential-required", failure.kind == "credential-required" and "ghcr.io" in failure.reason)
|
||||
check("permission denied on local disk is a plain failure", trivy.classify_failure("r", "open /cache/x: permission denied").kind == "failed")
|
||||
check("a pull timeout is failed, not credential-required", trivy.classify_failure("r", "context deadline exceeded").kind == "failed")
|
||||
|
||||
print("FAILED: " + str(len(FAILED)) if FAILED else "all checks passed")
|
||||
sys.exit(1 if FAILED else 0)
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the fixture-driven assertions in check.py. See chainsaw-test.yaml for the requirement tags.
|
||||
# What this does NOT prove: anything at runtime (real Trivy, real API, real browser). L2/L3 cover that.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
exec python3 "${HERE}/check.py"
|
||||
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"cronjobs": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "nightly",
|
||||
"namespace": "ops"
|
||||
},
|
||||
"spec": {
|
||||
"jobTemplate": {
|
||||
"spec": {
|
||||
"template": {
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/nightly:2.1",
|
||||
"name": "job"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"jobs": [],
|
||||
"nodes": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "prod-1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"pods": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "web-5f"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "app"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "app"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "api-77"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.27",
|
||||
"name": "server"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/nginx@sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"name": "server"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "db-0",
|
||||
"namespace": "data",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "StatefulSet",
|
||||
"name": "db"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "postgres:16",
|
||||
"name": "postgres"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "docker.io/library/postgres@sha256:d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4",
|
||||
"name": "postgres"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "priv-1",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "priv-1a"
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "ghcr.io/org/private:1",
|
||||
"name": "priv"
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"containerStatuses": [
|
||||
{
|
||||
"imageID": "ghcr.io/org/private@sha256:e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5",
|
||||
"name": "priv"
|
||||
}
|
||||
],
|
||||
"phase": "Running"
|
||||
}
|
||||
}
|
||||
],
|
||||
"replicasets": [
|
||||
{
|
||||
"metadata": {
|
||||
"name": "web-5f",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "web"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "api-77",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "api"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "priv-1a",
|
||||
"namespace": "apps",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"controller": true,
|
||||
"kind": "Deployment",
|
||||
"name": "priv"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"vmis": []
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"ArtifactName": "x",
|
||||
"SchemaVersion": 2
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"ArtifactName": "x",
|
||||
"Results": [
|
||||
{
|
||||
"Target": "t",
|
||||
"Vulnerabilities": [
|
||||
{
|
||||
"CVSS": {
|
||||
"nvd": {
|
||||
"V3Score": 9.8
|
||||
}
|
||||
},
|
||||
"FixedVersion": "3.0.2",
|
||||
"InstalledVersion": "1.0",
|
||||
"PkgName": "openssl",
|
||||
"PrimaryURL": "https://example.test/CVE-2026-0001",
|
||||
"Severity": "CRITICAL",
|
||||
"Title": "OpenSSL flaw",
|
||||
"VulnerabilityID": "CVE-2026-0001"
|
||||
},
|
||||
{
|
||||
"InstalledVersion": "1.0",
|
||||
"PkgName": "zlib",
|
||||
"PrimaryURL": "https://example.test/CVE-2026-0002",
|
||||
"Severity": "LOW",
|
||||
"Title": "t",
|
||||
"VulnerabilityID": "CVE-2026-0002"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SchemaVersion": 2
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"ArtifactName": "x",
|
||||
"Results": [
|
||||
{
|
||||
"Target": "t",
|
||||
"Vulnerabilities": [
|
||||
{
|
||||
"InstalledVersion": "1.0",
|
||||
"PkgName": "curl",
|
||||
"PrimaryURL": "https://example.test/CVE-2026-0020",
|
||||
"Severity": "MEDIUM",
|
||||
"Title": "t",
|
||||
"VulnerabilityID": "CVE-2026-0020"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SchemaVersion": 2
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"ArtifactName": "x",
|
||||
"Results": [
|
||||
{
|
||||
"Target": "t",
|
||||
"Vulnerabilities": [
|
||||
{
|
||||
"CVSS": {
|
||||
"nvd": {
|
||||
"V3Score": 7.5
|
||||
}
|
||||
},
|
||||
"FixedVersion": "16.2",
|
||||
"InstalledVersion": "1.0",
|
||||
"PkgName": "libpq",
|
||||
"PrimaryURL": "https://example.test/CVE-2026-0010",
|
||||
"Severity": "HIGH",
|
||||
"Title": "libpq flaw",
|
||||
"VulnerabilityID": "CVE-2026-0010"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SchemaVersion": 2
|
||||
}
|
||||
Reference in new issue
Block a user