jakman85andClaude Opus 5.5
50ec2297ee
Stdlib-only scanner per docs/17: Kubernetes API inventory, Trivy runner, persisted state, schema-validated report.json, strict acceptances parser, server-rendered page, aggregate metrics, internal API, and a scan loop that stays live while scanning. L1 tests under conformance/tests/l1-rendering/vuln-scanner-* run the actual modules against recorded fixtures with a fake Trivy runner. Co-Authored-By: Claude Opus 5.5 <[email protected]>
223 lines
9.3 KiB
Python
223 lines
9.3 KiB
Python
"""Build, validate and publish report.json, the one interface to the page and the bot."""
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import date, timedelta
|
|
|
|
import acceptances as acceptances_mod
|
|
import state as st
|
|
from state import fmt_ts, parse_ts
|
|
|
|
log = logging.getLogger("vuln-scanner.report")
|
|
|
|
STATES = ("scanned", "pending", "stale", "failed", "credential-required", "by-tag", "not-covered")
|
|
SCAN_WINDOW = timedelta(hours=24)
|
|
PENDING_WINDOW = timedelta(hours=2)
|
|
DB_STALE = timedelta(hours=48)
|
|
NOT_SCANNED_REASON = "not scanned within 2h of first appearing"
|
|
EPOCH = "1970-01-01T00:00:00Z" # database.updatedAt when no database has ever been downloaded
|
|
|
|
_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "report.schema.json")
|
|
with open(_SCHEMA_PATH, encoding="utf-8") as _handle:
|
|
SCHEMA = json.load(_handle)
|
|
|
|
|
|
def coverage(target, record, now):
|
|
"""(state, reason, failed-since datetime or None) for one target. The single place states come from."""
|
|
if target["type"] != "image":
|
|
return "not-covered", target["reason"], None
|
|
record = record or st.new_record(now)
|
|
outcome = record["outcome"]
|
|
if outcome == "credential-required":
|
|
return "credential-required", record["reason"], None
|
|
if outcome == "failed":
|
|
return "failed", record["reason"], parse_ts(record["failedSince"]) or now
|
|
if record["lastAttemptAt"] is None:
|
|
appeared = parse_ts(record["firstDiscoveredAt"]) or now
|
|
if now - appeared > PENDING_WINDOW:
|
|
return "failed", NOT_SCANNED_REASON, appeared + PENDING_WINDOW
|
|
return "pending", None, None
|
|
last_success = parse_ts(record["lastSuccessAt"]) or now
|
|
if now - last_success > SCAN_WINDOW:
|
|
return "stale", "last successful scan was at " + record["lastSuccessAt"], None
|
|
return ("by-tag" if target["digest"] is None else "scanned"), None, None
|
|
|
|
|
|
def failed_over_24h(state, now):
|
|
count = 0
|
|
for target in state["inventory"]:
|
|
status, _, since = coverage(target, state["scans"].get(st.scan_key(target)), now)
|
|
if status == "failed" and since is not None and now - since > SCAN_WINDOW:
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def build(state, accepted, acceptance_errors, now):
|
|
"""The report for `state` at `now`; acceptance_errors degrades the run."""
|
|
today = now.date()
|
|
targets = []
|
|
findings = []
|
|
for target in state["inventory"]:
|
|
record = state["scans"].get(st.scan_key(target))
|
|
status, reason, _ = coverage(target, record, now)
|
|
last_scanned = record["lastSuccessAt"] if record else None
|
|
targets.append({
|
|
"id": target["id"], "type": target["type"], "image": target["image"], "digest": target["digest"],
|
|
"state": status, "reason": reason, "lastScannedAt": last_scanned, "workloads": target["workloads"],
|
|
})
|
|
for item in (record or {}).get("findings", []):
|
|
finding = {key: item[key] for key in
|
|
("id", "severity", "cvss", "package", "installedVersion", "fixedVersion", "title", "url")}
|
|
finding["targetId"] = target["id"]
|
|
finding["firstSeenAt"] = record["firstSeen"].get(item["id"]) or fmt_ts(now)
|
|
acceptance = acceptances_mod.match(accepted, item["id"], target["image"], today)
|
|
if acceptance:
|
|
finding["status"] = "accepted"
|
|
finding["acceptance"] = {"reason": acceptance["reason"], "expires": acceptance["expires"].isoformat()}
|
|
else:
|
|
finding["status"] = "fix-available" if item["fixedVersion"] else "open"
|
|
findings.append(finding)
|
|
findings.sort(key=lambda f: (f["id"], f["targetId"], f["package"], f["installedVersion"]))
|
|
database = dict(state["database"]) if state["database"] else {"updatedAt": EPOCH, "version": "unavailable"}
|
|
run = dict(state["run"]) if state["run"] else {"startedAt": fmt_ts(now), "finishedAt": fmt_ts(now), "status": "degraded"}
|
|
db_time = parse_ts(database["updatedAt"])
|
|
if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
|
|
or db_time is None or now - db_time > DB_STALE:
|
|
run["status"] = "degraded"
|
|
return {
|
|
"schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
|
|
"targets": targets, "findings": findings, "remediation": list(state["remediation"]),
|
|
}
|
|
|
|
|
|
# --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
|
|
|
|
def _type_ok(value, name):
|
|
if name == "object":
|
|
return isinstance(value, dict)
|
|
if name == "array":
|
|
return isinstance(value, list)
|
|
if name == "string":
|
|
return isinstance(value, str)
|
|
if name == "null":
|
|
return value is None
|
|
if name == "boolean":
|
|
return isinstance(value, bool)
|
|
if name == "number":
|
|
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
if name == "integer":
|
|
return isinstance(value, int) and not isinstance(value, bool)
|
|
return False
|
|
|
|
|
|
def _format_ok(value, name):
|
|
if name == "date-time":
|
|
return parse_ts(value) is not None
|
|
if name == "date":
|
|
try:
|
|
date.fromisoformat(value)
|
|
return True
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _check(schema, value, path, errors):
|
|
if len(errors) >= 50:
|
|
return
|
|
if "type" in schema:
|
|
names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]]
|
|
if not any(_type_ok(value, n) for n in names):
|
|
errors.append(path + ": wrong type")
|
|
return
|
|
if "const" in schema and value != schema["const"]:
|
|
errors.append(path + ": not the required constant")
|
|
if "enum" in schema and value not in schema["enum"]:
|
|
errors.append(path + ": not an allowed value")
|
|
if isinstance(value, str):
|
|
if "minLength" in schema and len(value) < schema["minLength"]:
|
|
errors.append(path + ": too short")
|
|
if "maxLength" in schema and len(value) > schema["maxLength"]:
|
|
errors.append(path + ": too long")
|
|
if "pattern" in schema and not re.search(schema["pattern"], value):
|
|
errors.append(path + ": does not match the pattern")
|
|
if "format" in schema and not _format_ok(value, schema["format"]):
|
|
errors.append(path + ": not a valid " + schema["format"])
|
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
if "minimum" in schema and value < schema["minimum"]:
|
|
errors.append(path + ": below the minimum")
|
|
if "maximum" in schema and value > schema["maximum"]:
|
|
errors.append(path + ": above the maximum")
|
|
if isinstance(value, dict):
|
|
for key in schema.get("required", []):
|
|
if key not in value:
|
|
errors.append(path + ": missing " + key)
|
|
properties = schema.get("properties", {})
|
|
for key, item in value.items():
|
|
if key in properties:
|
|
_check(properties[key], item, path + "." + key, errors)
|
|
elif schema.get("additionalProperties") is False:
|
|
errors.append(path + ": unexpected key " + key)
|
|
if isinstance(value, list) and "items" in schema:
|
|
for index, item in enumerate(value):
|
|
_check(schema["items"], item, path + "[" + str(index) + "]", errors)
|
|
if "not" in schema:
|
|
trial = []
|
|
_check(schema["not"], value, path, trial)
|
|
if not trial:
|
|
errors.append(path + ": matches a forbidden shape")
|
|
if "if" in schema:
|
|
trial = []
|
|
_check(schema["if"], value, path, trial)
|
|
branch = schema.get("then") if not trial else schema.get("else")
|
|
if branch:
|
|
_check(branch, value, path, errors)
|
|
|
|
|
|
def validate(report):
|
|
"""Schema and referential errors; an empty list means the report may be served."""
|
|
errors = []
|
|
_check(SCHEMA, report, "$", errors)
|
|
if errors:
|
|
return errors
|
|
ids = [t["id"] for t in report["targets"]]
|
|
if len(set(ids)) != len(ids):
|
|
errors.append("$.targets: duplicate target id")
|
|
known = set(ids)
|
|
for finding in report["findings"]:
|
|
if finding["targetId"] not in known:
|
|
errors.append("$.findings: targetId has no target")
|
|
break
|
|
return errors
|
|
|
|
|
|
def serialise(report):
|
|
return json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def publish(report, data_dir):
|
|
"""Write report.json. An invalid report writes nothing and is logged loudly."""
|
|
errors = validate(report)
|
|
if errors:
|
|
log.critical("REFUSING to publish an invalid report; the previous one stands: %s", errors[:10])
|
|
return False
|
|
st.atomic_write(os.path.join(data_dir, "report.json"), serialise(report))
|
|
return True
|
|
|
|
|
|
def load_published(data_dir):
|
|
"""The published report if present and still valid, else None (never a corrupt report)."""
|
|
try:
|
|
with open(os.path.join(data_dir, "report.json"), "rb") as handle:
|
|
report = json.loads(handle.read())
|
|
except FileNotFoundError:
|
|
return None
|
|
except (OSError, ValueError):
|
|
log.critical("published report.json is unreadable; serving no report until the next publish")
|
|
return None
|
|
if validate(report):
|
|
log.critical("published report.json is invalid; serving no report until the next publish")
|
|
return None
|
|
return report
|