feat(vuln): scanner service and fixture-driven L1 tests
Stdlib-only scanner per docs/17: Kubernetes API inventory, Trivy runner, persisted state, schema-validated report.json, strict acceptances parser, server-rendered page, aggregate metrics, internal API, and a scan loop that stays live while scanning. L1 tests under conformance/tests/l1-rendering/vuln-scanner-* run the actual modules against recorded fixtures with a fake Trivy runner. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
4c3a284cad
commit
50ec2297ee
32 files changed
+3951
No files matched your search
@@ -0,0 +1,222 @@
|
||||
"""Build, validate and publish report.json, the one interface to the page and the bot."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
import acceptances as acceptances_mod
|
||||
import state as st
|
||||
from state import fmt_ts, parse_ts
|
||||
|
||||
log = logging.getLogger("vuln-scanner.report")
|
||||
|
||||
STATES = ("scanned", "pending", "stale", "failed", "credential-required", "by-tag", "not-covered")
|
||||
SCAN_WINDOW = timedelta(hours=24)
|
||||
PENDING_WINDOW = timedelta(hours=2)
|
||||
DB_STALE = timedelta(hours=48)
|
||||
NOT_SCANNED_REASON = "not scanned within 2h of first appearing"
|
||||
EPOCH = "1970-01-01T00:00:00Z" # database.updatedAt when no database has ever been downloaded
|
||||
|
||||
_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "report.schema.json")
|
||||
with open(_SCHEMA_PATH, encoding="utf-8") as _handle:
|
||||
SCHEMA = json.load(_handle)
|
||||
|
||||
|
||||
def coverage(target, record, now):
|
||||
"""(state, reason, failed-since datetime or None) for one target. The single place states come from."""
|
||||
if target["type"] != "image":
|
||||
return "not-covered", target["reason"], None
|
||||
record = record or st.new_record(now)
|
||||
outcome = record["outcome"]
|
||||
if outcome == "credential-required":
|
||||
return "credential-required", record["reason"], None
|
||||
if outcome == "failed":
|
||||
return "failed", record["reason"], parse_ts(record["failedSince"]) or now
|
||||
if record["lastAttemptAt"] is None:
|
||||
appeared = parse_ts(record["firstDiscoveredAt"]) or now
|
||||
if now - appeared > PENDING_WINDOW:
|
||||
return "failed", NOT_SCANNED_REASON, appeared + PENDING_WINDOW
|
||||
return "pending", None, None
|
||||
last_success = parse_ts(record["lastSuccessAt"]) or now
|
||||
if now - last_success > SCAN_WINDOW:
|
||||
return "stale", "last successful scan was at " + record["lastSuccessAt"], None
|
||||
return ("by-tag" if target["digest"] is None else "scanned"), None, None
|
||||
|
||||
|
||||
def failed_over_24h(state, now):
|
||||
count = 0
|
||||
for target in state["inventory"]:
|
||||
status, _, since = coverage(target, state["scans"].get(st.scan_key(target)), now)
|
||||
if status == "failed" and since is not None and now - since > SCAN_WINDOW:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def build(state, accepted, acceptance_errors, now):
|
||||
"""The report for `state` at `now`; acceptance_errors degrades the run."""
|
||||
today = now.date()
|
||||
targets = []
|
||||
findings = []
|
||||
for target in state["inventory"]:
|
||||
record = state["scans"].get(st.scan_key(target))
|
||||
status, reason, _ = coverage(target, record, now)
|
||||
last_scanned = record["lastSuccessAt"] if record else None
|
||||
targets.append({
|
||||
"id": target["id"], "type": target["type"], "image": target["image"], "digest": target["digest"],
|
||||
"state": status, "reason": reason, "lastScannedAt": last_scanned, "workloads": target["workloads"],
|
||||
})
|
||||
for item in (record or {}).get("findings", []):
|
||||
finding = {key: item[key] for key in
|
||||
("id", "severity", "cvss", "package", "installedVersion", "fixedVersion", "title", "url")}
|
||||
finding["targetId"] = target["id"]
|
||||
finding["firstSeenAt"] = record["firstSeen"].get(item["id"]) or fmt_ts(now)
|
||||
acceptance = acceptances_mod.match(accepted, item["id"], target["image"], today)
|
||||
if acceptance:
|
||||
finding["status"] = "accepted"
|
||||
finding["acceptance"] = {"reason": acceptance["reason"], "expires": acceptance["expires"].isoformat()}
|
||||
else:
|
||||
finding["status"] = "fix-available" if item["fixedVersion"] else "open"
|
||||
findings.append(finding)
|
||||
findings.sort(key=lambda f: (f["id"], f["targetId"], f["package"], f["installedVersion"]))
|
||||
database = dict(state["database"]) if state["database"] else {"updatedAt": EPOCH, "version": "unavailable"}
|
||||
run = dict(state["run"]) if state["run"] else {"startedAt": fmt_ts(now), "finishedAt": fmt_ts(now), "status": "degraded"}
|
||||
db_time = parse_ts(database["updatedAt"])
|
||||
if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
|
||||
or db_time is None or now - db_time > DB_STALE:
|
||||
run["status"] = "degraded"
|
||||
return {
|
||||
"schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
|
||||
"targets": targets, "findings": findings, "remediation": list(state["remediation"]),
|
||||
}
|
||||
|
||||
|
||||
# --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
|
||||
|
||||
def _type_ok(value, name):
|
||||
if name == "object":
|
||||
return isinstance(value, dict)
|
||||
if name == "array":
|
||||
return isinstance(value, list)
|
||||
if name == "string":
|
||||
return isinstance(value, str)
|
||||
if name == "null":
|
||||
return value is None
|
||||
if name == "boolean":
|
||||
return isinstance(value, bool)
|
||||
if name == "number":
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
if name == "integer":
|
||||
return isinstance(value, int) and not isinstance(value, bool)
|
||||
return False
|
||||
|
||||
|
||||
def _format_ok(value, name):
|
||||
if name == "date-time":
|
||||
return parse_ts(value) is not None
|
||||
if name == "date":
|
||||
try:
|
||||
date.fromisoformat(value)
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _check(schema, value, path, errors):
|
||||
if len(errors) >= 50:
|
||||
return
|
||||
if "type" in schema:
|
||||
names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]]
|
||||
if not any(_type_ok(value, n) for n in names):
|
||||
errors.append(path + ": wrong type")
|
||||
return
|
||||
if "const" in schema and value != schema["const"]:
|
||||
errors.append(path + ": not the required constant")
|
||||
if "enum" in schema and value not in schema["enum"]:
|
||||
errors.append(path + ": not an allowed value")
|
||||
if isinstance(value, str):
|
||||
if "minLength" in schema and len(value) < schema["minLength"]:
|
||||
errors.append(path + ": too short")
|
||||
if "maxLength" in schema and len(value) > schema["maxLength"]:
|
||||
errors.append(path + ": too long")
|
||||
if "pattern" in schema and not re.search(schema["pattern"], value):
|
||||
errors.append(path + ": does not match the pattern")
|
||||
if "format" in schema and not _format_ok(value, schema["format"]):
|
||||
errors.append(path + ": not a valid " + schema["format"])
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
if "minimum" in schema and value < schema["minimum"]:
|
||||
errors.append(path + ": below the minimum")
|
||||
if "maximum" in schema and value > schema["maximum"]:
|
||||
errors.append(path + ": above the maximum")
|
||||
if isinstance(value, dict):
|
||||
for key in schema.get("required", []):
|
||||
if key not in value:
|
||||
errors.append(path + ": missing " + key)
|
||||
properties = schema.get("properties", {})
|
||||
for key, item in value.items():
|
||||
if key in properties:
|
||||
_check(properties[key], item, path + "." + key, errors)
|
||||
elif schema.get("additionalProperties") is False:
|
||||
errors.append(path + ": unexpected key " + key)
|
||||
if isinstance(value, list) and "items" in schema:
|
||||
for index, item in enumerate(value):
|
||||
_check(schema["items"], item, path + "[" + str(index) + "]", errors)
|
||||
if "not" in schema:
|
||||
trial = []
|
||||
_check(schema["not"], value, path, trial)
|
||||
if not trial:
|
||||
errors.append(path + ": matches a forbidden shape")
|
||||
if "if" in schema:
|
||||
trial = []
|
||||
_check(schema["if"], value, path, trial)
|
||||
branch = schema.get("then") if not trial else schema.get("else")
|
||||
if branch:
|
||||
_check(branch, value, path, errors)
|
||||
|
||||
|
||||
def validate(report):
|
||||
"""Schema and referential errors; an empty list means the report may be served."""
|
||||
errors = []
|
||||
_check(SCHEMA, report, "$", errors)
|
||||
if errors:
|
||||
return errors
|
||||
ids = [t["id"] for t in report["targets"]]
|
||||
if len(set(ids)) != len(ids):
|
||||
errors.append("$.targets: duplicate target id")
|
||||
known = set(ids)
|
||||
for finding in report["findings"]:
|
||||
if finding["targetId"] not in known:
|
||||
errors.append("$.findings: targetId has no target")
|
||||
break
|
||||
return errors
|
||||
|
||||
|
||||
def serialise(report):
|
||||
return json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def publish(report, data_dir):
|
||||
"""Write report.json. An invalid report writes nothing and is logged loudly."""
|
||||
errors = validate(report)
|
||||
if errors:
|
||||
log.critical("REFUSING to publish an invalid report; the previous one stands: %s", errors[:10])
|
||||
return False
|
||||
st.atomic_write(os.path.join(data_dir, "report.json"), serialise(report))
|
||||
return True
|
||||
|
||||
|
||||
def load_published(data_dir):
|
||||
"""The published report if present and still valid, else None (never a corrupt report)."""
|
||||
try:
|
||||
with open(os.path.join(data_dir, "report.json"), "rb") as handle:
|
||||
report = json.loads(handle.read())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (OSError, ValueError):
|
||||
log.critical("published report.json is unreadable; serving no report until the next publish")
|
||||
return None
|
||||
if validate(report):
|
||||
log.critical("published report.json is invalid; serving no report until the next publish")
|
||||
return None
|
||||
return report
|
||||
Reference in new issue
Block a user