jakman85andClaude Opus 5.5
d8b25a67fe
Review findings on the integrated scanner: - An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart that image is marked failed and sits out full runs for 20h. - One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a report is actually published. - The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names it, and Trivy is not invoked. - loadError clears after a full run against a good inventory. - Every image under platform/vuln-scanner stays out of Renovate auto-merge. - The page labels status and target state per line, and mixed acceptance reads as mixed. The Keycloak client is still an operator action: its key must go in an existing encrypted file that an agent must not modify. platform/vuln-scanner/README.md has the exact steps. Co-Authored-By: Claude Opus 5.5 <[email protected]>
311 lines
13 KiB
Python
311 lines
13 KiB
Python
"""Build, validate and publish report.json, the one interface to the page and the bot."""
|
|
import json
|
|
import logging
|
|
import hashlib
|
|
import os
|
|
import re
|
|
from datetime import date, timedelta
|
|
|
|
import acceptances as acceptances_mod
|
|
import state as st
|
|
from state import fmt_ts, parse_ts
|
|
|
|
log = logging.getLogger("vuln-scanner.report")
|
|
|
|
STATES = ("scanned", "pending", "stale", "failed", "credential-required", "by-tag", "not-covered")
|
|
SCAN_WINDOW = timedelta(hours=24)
|
|
PENDING_WINDOW = timedelta(hours=2)
|
|
DB_STALE = timedelta(hours=48)
|
|
NOT_SCANNED_REASON = "not scanned within 2h of first appearing"
|
|
EPOCH = "1970-01-01T00:00:00Z" # database.updatedAt when no database has ever been downloaded
|
|
|
|
_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "report.schema.json")
|
|
with open(_SCHEMA_PATH, encoding="utf-8") as _handle:
|
|
SCHEMA = json.load(_handle)
|
|
|
|
|
|
def coverage(target, record, now):
|
|
"""(state, reason, failed-since datetime or None) for one target. The single place states come from."""
|
|
if target["type"] != "image":
|
|
return "not-covered", target["reason"], None
|
|
record = record or st.new_record(now)
|
|
outcome = record["outcome"]
|
|
if outcome == "credential-required":
|
|
return "credential-required", record["reason"], None
|
|
if outcome == "failed":
|
|
return "failed", record["reason"], parse_ts(record["failedSince"]) or now
|
|
if record["lastAttemptAt"] is None:
|
|
appeared = parse_ts(record["firstDiscoveredAt"]) or now
|
|
if now - appeared > PENDING_WINDOW:
|
|
return "failed", NOT_SCANNED_REASON, appeared + PENDING_WINDOW
|
|
return "pending", None, None
|
|
last_success = parse_ts(record["lastSuccessAt"]) or now
|
|
if now - last_success > SCAN_WINDOW:
|
|
return "stale", "last successful scan was at " + record["lastSuccessAt"], None
|
|
return ("by-tag" if target["digest"] is None else "scanned"), None, None
|
|
|
|
|
|
def failed_over_24h(state, now):
|
|
count = 0
|
|
for target in state["inventory"]:
|
|
status, _, since = coverage(target, state["scans"].get(st.scan_key(target)), now)
|
|
if status == "failed" and since is not None and now - since > SCAN_WINDOW:
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def build(state, accepted, acceptance_errors, now):
|
|
"""The report for `state` at `now`; acceptance_errors degrades the run."""
|
|
today = now.date()
|
|
targets = []
|
|
findings = []
|
|
for target in state["inventory"]:
|
|
record = state["scans"].get(st.scan_key(target))
|
|
status, reason, _ = coverage(target, record, now)
|
|
last_scanned = record["lastSuccessAt"] if record else None
|
|
targets.append({
|
|
"id": target["id"], "type": target["type"], "image": target["image"], "digest": target["digest"],
|
|
"state": status, "reason": reason, "lastScannedAt": last_scanned, "workloads": target["workloads"],
|
|
})
|
|
for item in (record or {}).get("findings", []):
|
|
finding = {key: item[key] for key in
|
|
("id", "severity", "cvss", "package", "installedVersion", "fixedVersion", "title", "url")}
|
|
finding["targetId"] = target["id"]
|
|
finding["firstSeenAt"] = record["firstSeen"].get(item["id"]) or fmt_ts(now)
|
|
acceptance = acceptances_mod.match(accepted, item["id"], target["image"], today)
|
|
if acceptance:
|
|
finding["status"] = "accepted"
|
|
finding["acceptance"] = {"reason": acceptance["reason"], "expires": acceptance["expires"].isoformat()}
|
|
else:
|
|
finding["status"] = "fix-available" if item["fixedVersion"] else "open"
|
|
findings.append(finding)
|
|
findings.sort(key=lambda f: (f["id"], f["targetId"], f["package"], f["installedVersion"]))
|
|
database = dict(state["database"]) if state["database"] else {"updatedAt": EPOCH, "version": "unavailable"}
|
|
run = dict(state["run"]) if state["run"] else {"startedAt": fmt_ts(now), "finishedAt": fmt_ts(now), "status": "degraded"}
|
|
db_time = parse_ts(database["updatedAt"])
|
|
if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
|
|
or db_time is None or now - db_time > DB_STALE:
|
|
run["status"] = "degraded"
|
|
built = {
|
|
"schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
|
|
"targets": targets, "findings": findings, "remediation": list(state["remediation"]),
|
|
}
|
|
repaired = _isolate(built)
|
|
if repaired:
|
|
log.error("%d report entries held a value the schema refuses; repaired or dropped them", repaired)
|
|
run["status"] = "degraded"
|
|
return built
|
|
|
|
|
|
# --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
|
|
|
|
def _type_ok(value, name):
|
|
if name == "object":
|
|
return isinstance(value, dict)
|
|
if name == "array":
|
|
return isinstance(value, list)
|
|
if name == "string":
|
|
return isinstance(value, str)
|
|
if name == "null":
|
|
return value is None
|
|
if name == "boolean":
|
|
return isinstance(value, bool)
|
|
if name == "number":
|
|
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
if name == "integer":
|
|
return isinstance(value, int) and not isinstance(value, bool)
|
|
return False
|
|
|
|
|
|
def _format_ok(value, name):
|
|
if name == "date-time":
|
|
return parse_ts(value) is not None
|
|
if name == "date":
|
|
try:
|
|
date.fromisoformat(value)
|
|
return True
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _check(schema, value, path, errors):
|
|
if len(errors) >= 50:
|
|
return
|
|
if "type" in schema:
|
|
names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]]
|
|
if not any(_type_ok(value, n) for n in names):
|
|
errors.append(path + ": wrong type")
|
|
return
|
|
if "const" in schema and value != schema["const"]:
|
|
errors.append(path + ": not the required constant")
|
|
if "enum" in schema and value not in schema["enum"]:
|
|
errors.append(path + ": not an allowed value")
|
|
if isinstance(value, str):
|
|
if "minLength" in schema and len(value) < schema["minLength"]:
|
|
errors.append(path + ": too short")
|
|
if "maxLength" in schema and len(value) > schema["maxLength"]:
|
|
errors.append(path + ": too long")
|
|
if "pattern" in schema and not re.search(schema["pattern"], value):
|
|
errors.append(path + ": does not match the pattern")
|
|
if "format" in schema and not _format_ok(value, schema["format"]):
|
|
errors.append(path + ": not a valid " + schema["format"])
|
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
if "minimum" in schema and value < schema["minimum"]:
|
|
errors.append(path + ": below the minimum")
|
|
if "maximum" in schema and value > schema["maximum"]:
|
|
errors.append(path + ": above the maximum")
|
|
if isinstance(value, dict):
|
|
for key in schema.get("required", []):
|
|
if key not in value:
|
|
errors.append(path + ": missing " + key)
|
|
properties = schema.get("properties", {})
|
|
for key, item in value.items():
|
|
if key in properties:
|
|
_check(properties[key], item, path + "." + key, errors)
|
|
elif schema.get("additionalProperties") is False:
|
|
errors.append(path + ": unexpected key " + key)
|
|
if isinstance(value, list) and "items" in schema:
|
|
for index, item in enumerate(value):
|
|
_check(schema["items"], item, path + "[" + str(index) + "]", errors)
|
|
if "not" in schema:
|
|
trial = []
|
|
_check(schema["not"], value, path, trial)
|
|
if not trial:
|
|
errors.append(path + ": matches a forbidden shape")
|
|
if "if" in schema:
|
|
trial = []
|
|
_check(schema["if"], value, path, trial)
|
|
branch = schema.get("then") if not trial else schema.get("else")
|
|
if branch:
|
|
_check(branch, value, path, errors)
|
|
|
|
|
|
# --- isolation: one bad upstream value costs that value, never the whole report ---------------------
|
|
|
|
def _repair(schema, value):
|
|
"""(usable, value): value itself when valid, else the nearest valid form of it."""
|
|
trial = []
|
|
_check(schema, value, "$", trial)
|
|
if not trial:
|
|
return True, value
|
|
if isinstance(value, dict) and "properties" in schema:
|
|
fixed = {}
|
|
for key, sub in schema["properties"].items():
|
|
if key not in value:
|
|
continue
|
|
usable, item = _repair(sub, value[key])
|
|
if usable:
|
|
fixed[key] = item
|
|
elif key in schema.get("required", []):
|
|
return False, None
|
|
trial = []
|
|
_check(schema, fixed, "$", trial)
|
|
return not trial, fixed
|
|
if isinstance(value, list) and "items" in schema:
|
|
repaired = [_repair(schema["items"], item) for item in value]
|
|
return True, [item for usable, item in repaired if usable]
|
|
candidates = [None, "", "UNKNOWN"]
|
|
if isinstance(value, str) and "maxLength" in schema:
|
|
candidates.insert(0, value[:schema["maxLength"]])
|
|
for candidate in candidates:
|
|
trial = []
|
|
_check(schema, candidate, "$", trial)
|
|
if not trial:
|
|
return True, candidate
|
|
return False, None
|
|
|
|
|
|
def _isolate(report):
|
|
"""Repair (or, as a last resort, drop) each entry the schema refuses. Returns how many changed."""
|
|
changed = 0
|
|
schema = SCHEMA["properties"]
|
|
usable, database = _repair(schema["database"], report["database"])
|
|
if database is not report["database"]:
|
|
changed += 1
|
|
report["database"] = database if usable else {"updatedAt": EPOCH, "version": "unavailable"}
|
|
renamed, dropped, targets = {}, set(), []
|
|
for target in report["targets"]:
|
|
original = target["id"]
|
|
if len(original) > 2048: # keep it unique: a truncated id may not collide with another
|
|
target = dict(target, id=original[:2031] + "#" + hashlib.sha256(original.encode("utf-8")).hexdigest()[:16])
|
|
renamed[original] = target["id"]
|
|
usable, fixed = _repair(schema["targets"]["items"], target)
|
|
if not usable:
|
|
changed += 1
|
|
dropped.add(original)
|
|
continue
|
|
if fixed is not target or original in renamed:
|
|
changed += 1
|
|
targets.append(fixed)
|
|
report["targets"] = targets
|
|
findings = []
|
|
for finding in report["findings"]:
|
|
if finding["targetId"] in dropped:
|
|
changed += 1
|
|
continue
|
|
if finding["targetId"] in renamed:
|
|
finding = dict(finding, targetId=renamed[finding["targetId"]])
|
|
usable, fixed = _repair(schema["findings"]["items"], finding)
|
|
if not usable or fixed is not finding:
|
|
changed += 1
|
|
if usable:
|
|
findings.append(fixed)
|
|
report["findings"] = findings
|
|
remediation = []
|
|
for entry in report["remediation"]:
|
|
usable, fixed = _repair(schema["remediation"]["items"], entry)
|
|
if not usable or fixed is not entry:
|
|
changed += 1
|
|
if usable:
|
|
remediation.append(fixed)
|
|
report["remediation"] = remediation
|
|
return changed
|
|
|
|
|
|
def validate(report):
|
|
"""Schema and referential errors; an empty list means the report may be served."""
|
|
errors = []
|
|
_check(SCHEMA, report, "$", errors)
|
|
if errors:
|
|
return errors
|
|
ids = [t["id"] for t in report["targets"]]
|
|
if len(set(ids)) != len(ids):
|
|
errors.append("$.targets: duplicate target id")
|
|
known = set(ids)
|
|
for finding in report["findings"]:
|
|
if finding["targetId"] not in known:
|
|
errors.append("$.findings: targetId has no target")
|
|
break
|
|
return errors
|
|
|
|
|
|
def serialise(report):
|
|
return json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def publish(report, data_dir):
|
|
"""Write report.json. An invalid report writes nothing and is logged loudly."""
|
|
errors = validate(report)
|
|
if errors:
|
|
log.critical("REFUSING to publish an invalid report; the previous one stands: %s", errors[:10])
|
|
return False
|
|
st.atomic_write(os.path.join(data_dir, "report.json"), serialise(report))
|
|
return True
|
|
|
|
|
|
def load_published(data_dir):
|
|
"""The published report if present and still valid, else None (never a corrupt report)."""
|
|
try:
|
|
with open(os.path.join(data_dir, "report.json"), "rb") as handle:
|
|
report = json.loads(handle.read())
|
|
except FileNotFoundError:
|
|
return None
|
|
except (OSError, ValueError):
|
|
log.critical("published report.json is unreadable; serving no report until the next publish")
|
|
return None
|
|
if validate(report):
|
|
log.critical("published report.json is invalid; serving no report until the next publish")
|
|
return None
|
|
return report
|