fix(vuln): survive OOM kills, isolate bad upstream strings, label findings per line

Review findings on the integrated scanner:
- An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart
  that image is marked failed and sits out full runs for 20h.
- One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase
  scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build
  repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a
  report is actually published.
- The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names
  it, and Trivy is not invoked.
- loadError clears after a full run against a good inventory.
- Every image under platform/vuln-scanner stays out of Renovate auto-merge.
- The page labels status and target state per line, and mixed acceptance reads as mixed.

The Keycloak client is still an operator action: its key must go in an existing encrypted file
that an agent must not modify. platform/vuln-scanner/README.md has the exact steps.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

platform/vuln-scanner/src/acceptances.py

@@ -15,6 +15,7 @@ from datetime import date, timedelta
 from state import utcnow
 
 MAX_DAYS = 180
+MAX_REASON = 2048  # report.schema.json's limit; a longer reason would make the whole report invalid
 _KEYS = ("id", "image", "reason", "expires")
 _ENTRY = re.compile(r"^- ([a-z]+):(?: (.*))?$")
 _FIELD = re.compile(r"^  ([a-z]+):(?: (.*))?$")
@@ -91,6 +92,8 @@ def _validate(fields, today):
         problems.append("id is missing or malformed")
     if not fields.get("reason"):
         problems.append("reason is mandatory")
+    elif len(fields["reason"]) > MAX_REASON:
+        problems.append("reason is longer than " + str(MAX_REASON) + " characters")
     if "image" in fields and not fields["image"]:
         problems.append("image is empty")
     expires = fields.get("expires") or ""

platform/vuln-scanner/src/main.py

@@ -23,15 +23,21 @@ log = logging.getLogger("vuln-scanner")
 FULL_RUN_INTERVAL = timedelta(hours=20)
 DB_MAX_AGE = timedelta(hours=48)
 DB_REFRESH_AGE = timedelta(hours=24)
+KILLED_REASON = "scanner killed during scan (OOM/eviction?)"
 
 
 class Engine:
-    def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120):
+    def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120,
+                 private_registries=()):
         self.store = st.Store(data_dir)
         self.data_dir = data_dir
         self.client = client
         self.runner = runner
         self.acceptances_path = acceptances_path
+        # Registries known to need credentials the scanner deliberately does not hold (R-VULN-09) and,
+        # being on the LAN, not reachable from its egress policy: a scan would end in a connection
+        # error (`failed`) rather than a 401 (`credential-required`), so it is never attempted.
+        self.private_registries = {name.strip().lower() for name in private_registries if name.strip()}
         self.clock = clock
         self.tick_seconds = tick_seconds
         self.publish_seconds = publish_seconds
@@ -43,6 +49,18 @@ class Engine:
         self.loop_thread = None
         self._last_publish = 0.0
         self._last_inventory = None
+        self._recover_interrupted_scan()
+
+    def _recover_interrupted_scan(self):
+        """An inFlight key at startup means the last process died inside that scan."""
+        key = self.state.get("inFlight")
+        if not key:
+            return
+        log.error("the previous process was killed while scanning %s; marking it failed", key)
+        record = self.state["scans"].get(key)
+        if record is not None:
+            record["killedAt"] = fmt_ts(self.clock())
+        self._record_failure(key, "failed", KILLED_REASON)
 
     # --- views used by the HTTP servers; none of them waits for a scan --------------------------
     def report_bytes(self):
@@ -67,16 +85,19 @@ class Engine:
 
     # --- persistence -----------------------------------------------------------------------------
     def publish(self):
+        """True when the report was written; False when it was refused and the previous one stands."""
         with self.lock:
             now = self.clock()
             entries, errors = acceptances.load(self.acceptances_path, now.date())
             for problem in errors:
                 log.error("invalid acceptance ignored: %s", problem)
             built = report_mod.build(self.state, entries, errors, now)
-            if report_mod.publish(built, self.data_dir):
+            published = report_mod.publish(built, self.data_dir)
+            if published:
                 self.report = built
                 self.report_json = report_mod.serialise(built)
             self._last_publish = time.monotonic()
+            return published
 
     # --- inventory -------------------------------------------------------------------------------
     def refresh_inventory(self):
@@ -152,6 +173,14 @@ class Engine:
         return timedelta.max if moment is None else self.clock() - moment
 
     def _scan_unit(self, key, reference):
+        registry = trivy.registry_of(reference)
+        if registry in self.private_registries or registry.split(":", 1)[0] in self.private_registries:
+            self._record_failure(key, "credential-required",
+                                 "registry " + registry + " is private and the scanner holds no credential for it")
+            return
+        with self.lock:
+            self.state["inFlight"] = key
+            self.store.save(self.state)
         try:
             found = trivy.parse_results(self.runner.scan(reference))
         except trivy.ScanFailure as exc:
@@ -166,11 +195,24 @@ class Engine:
         except Exception:  # pylint: disable=broad-except
             log.exception("scan cache cleanup failed")
 
+    def _finish_in_flight(self, key):
+        """Cleared in the same write as the scan's outcome, so no crash can separate the two."""
+        if self.state.get("inFlight") == key:
+            self.state["inFlight"] = None
+
+    def _killed_recently(self, key):
+        """A unit that killed the scanner sits out full runs for a day: retrying it first on every
+        restart would OOM again before any later image, which is the wedge this exists to prevent."""
+        killed = parse_ts(self.state["scans"][key].get("killedAt"))
+        return killed is not None and self.clock() - killed < FULL_RUN_INTERVAL
+
     def _record_failure(self, key, kind, reason):
         now = fmt_ts(self.clock())
         with self.lock:
+            self._finish_in_flight(key)
             record = self.state["scans"].get(key)
             if record is None:
+                self.store.save(self.state)
                 return
             record["lastAttemptAt"] = now
             record["outcome"] = kind
@@ -181,10 +223,13 @@ class Engine:
     def _record_success(self, key, found):
         now = fmt_ts(self.clock())
         with self.lock:
+            self._finish_in_flight(key)
             record = self.state["scans"].get(key)
             if record is None:
+                self.store.save(self.state)
                 return
             record["lastAttemptAt"] = now
+            record["killedAt"] = None
             record["lastSuccessAt"] = now
             record["outcome"] = "ok"
             record["reason"] = None
@@ -215,7 +260,10 @@ class Engine:
         started = self.clock()
         log.info("full run starting")
         database_ok = self._refresh_database(always=True)
-        self._scan_queue(sorted(self._units()))
+        self._scan_queue([key for key in sorted(self._units()) if not self._killed_recently(key)])
+        if self.stop.is_set():
+            log.info("full run interrupted by shutdown; not recorded")
+            return
         info = self.state["database"]
         now = self.clock()
         degraded = not database_ok or info is None or self._db_age(info) > DB_MAX_AGE
@@ -225,8 +273,15 @@ class Engine:
         status = "degraded" if degraded else "complete"
         with self.lock:
             self.state["run"] = {"startedAt": fmt_ts(started), "finishedAt": fmt_ts(now), "status": status}
-            if not degraded:
+            self.store.save(self.state)
+            # Only a report that actually reached the store counts as a completed run. Otherwise a
+            # refused report would freeze the page while VulnScanStale stayed quiet.
+            if self.publish() and not degraded and self.report["run"]["status"] == "complete":
                 self.state["lastCompleteRunAt"] = fmt_ts(now)
+            # A corrupt state.json degrades the run that rebuilds from it, and no further: once a full
+            # run has completed against a good inventory, the state is whole again.
+            if self.state.get("loadError") and self._last_inventory is not None and not self.state.get("inventoryError"):
+                self.state["loadError"] = None
             self.store.save(self.state)
         log.info("full run finished: %s", status)
 
@@ -262,7 +317,9 @@ def main():
     tools_dir = os.environ.get("VULN_TOOLS_DIR", "/tools")
     trivy_bin = os.environ.get("TRIVY_BIN", os.path.join(tools_dir, "trivy"))
     runner = trivy.Runner(trivy_bin, cache_dir, int(os.environ.get("VULN_SCAN_TIMEOUT_SECONDS", "600")))
-    engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"))
+    private_registries = os.environ.get("VULN_PRIVATE_REGISTRIES", "").split(",")
+    engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"),
+                    private_registries=private_registries)
     server.start_all(engine, ("127.0.0.1", 8080), ("0.0.0.0", 8081), ("0.0.0.0", 9090))
     finished = threading.Event()
     signal.signal(signal.SIGTERM, lambda *_: finished.set())

platform/vuln-scanner/src/page.py

@@ -180,14 +180,39 @@ def _sev(severity):
     return '<span class="sev-' + e(severity) + '">' + e(severity) + "</span>"
 
 
-def _workload_lines(target):
+def _state_label(target):
+    return "target " + e(target["state"]) + (" (" + e(target["reason"]) + ")" if target.get("reason") else "")
+
+
+def _status_label(finding):
+    """One finding's own status, with its own acceptance: never another image's."""
+    label = "status " + e(finding["status"])
+    acceptance = finding.get("acceptance")
+    if acceptance:
+        label += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
+    return label
+
+
+def _workload_lines(target, finding=None):
+    """One line per workload, each carrying the target state and, in a CVE expansion, the status."""
+    suffix = " — " + _state_label(target) + (" — " + _status_label(finding) if finding else "")
     items = []
     for workload in target["workloads"]:
         items.append("<li>" + e(workload["namespace"]) + " / " + e(workload["kind"]) + " / " + e(workload["name"])
-                     + " — containers: " + e(", ".join(workload["containers"]) or "none") + "</li>")
+                     + " — containers: " + e(", ".join(workload["containers"]) or "none") + suffix + "</li>")
     return "<ul>" + "".join(items) + "</ul>"
 
 
+def _status_summary(statuses):
+    """The row-level status. A CVE accepted on one image and open on another reads as mixed."""
+    counts = {}
+    for status in statuses:
+        counts[status] = counts.get(status, 0) + 1
+    if len(counts) == 1:
+        return e(next(iter(counts)))
+    return "mixed: " + ", ".join(e(counts[s]) + " " + e(s) for s in sorted(counts, key=STATUS_RANK.get))
+
+
 def _remediation_cell(images, remediation):
     if not images:
         return ""
@@ -250,13 +275,10 @@ def _cves(report, targets, params, remediation):
         for finding, target in row["items"]:
             if finding["status"] == "fix-available":
                 fix_images.add(target["image"])
-            lines.append("<li>" + e(target["id"]) + " [" + e(target["state"]) + "] — " + e(finding["package"]) + " "
-                         + e(finding["installedVersion"]) + " → " + e(finding["fixedVersion"] or "no fix")
-                         + _workload_lines(target) + "</li>")
-        acceptance = next((f.get("acceptance") for f, _ in row["items"] if f.get("acceptance")), None)
-        status_text = e(row["status"])
-        if acceptance:
-            status_text += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
+            lines.append("<li>" + e(target["id"]) + " — " + _state_label(target) + " — " + _status_label(finding)
+                         + " — " + e(finding["package"]) + " " + e(finding["installedVersion"]) + " → "
+                         + e(finding["fixedVersion"] or "no fix") + _workload_lines(target, finding) + "</li>")
+        status_text = _status_summary(f["status"] for f, _ in row["items"])
         html_rows.append(
             "<tr><td>" + _link(row["url"], row["id"]) + "</td><td>" + _sev(row["severity"]) + "</td><td>"
             + e("" if row["cvss"] is None else row["cvss"]) + "</td><td>" + e(row["package"]) + "</td><td>" + e(row["fix"])
@@ -317,12 +339,13 @@ def _workloads(report, targets, params):
     html_rows = []
     for row in page_rows:
         lines = "".join(
-            "<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + e(f["status"]) + "</li>"
-            for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"])))
+            "<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + _status_label(f)
+            + " — " + e(f["targetId"]) + " — " + _state_label(targets[f["targetId"]]) + "</li>"
+            for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"], f["targetId"])))
         counts = "".join("<td>" + e(row["counts"][s]) + "</td>" for s in SEVERITIES)
         html_rows.append(
             "<tr><td>" + e(row["key"][0]) + "</td><td>" + e(row["key"][1]) + "</td><td>" + e(row["key"][2]) + "</td><td>"
-            + e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) for i in sorted(row["images"])) + "</td>"
+            + e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) + " — " + _state_label(targets[i]) for i in sorted(row["images"])) + "</td>"
             + counts + "<td><details><summary>" + e(len(row["findings"])) + " findings</summary><ul>" + lines + "</ul></details></td></tr>")
     head = [_th(params, "Namespace", "namespace"), _th(params, "Kind", "kind"), _th(params, "Workload", "name"), "<th>Containers</th>",
             _th(params, "Images", "images")] + [_th(params, s, s.lower()) for s in SEVERITIES] + [_th(params, "Findings", "total")]

platform/vuln-scanner/src/report.py

@@ -1,6 +1,7 @@
 """Build, validate and publish report.json, the one interface to the page and the bot."""
 import json
 import logging
+import hashlib
 import os
 import re
 from datetime import date, timedelta
@@ -85,10 +86,15 @@ def build(state, accepted, acceptance_errors, now):
     if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
             or db_time is None or now - db_time > DB_STALE:
         run["status"] = "degraded"
-    return {
+    built = {
         "schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
         "targets": targets, "findings": findings, "remediation": list(state["remediation"]),
     }
+    repaired = _isolate(built)
+    if repaired:
+        log.error("%d report entries held a value the schema refuses; repaired or dropped them", repaired)
+        run["status"] = "degraded"
+    return built
 
 
 # --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
@@ -175,6 +181,88 @@ def _check(schema, value, path, errors):
             _check(branch, value, path, errors)
 
 
+# --- isolation: one bad upstream value costs that value, never the whole report ---------------------
+
+def _repair(schema, value):
+    """(usable, value): value itself when valid, else the nearest valid form of it."""
+    trial = []
+    _check(schema, value, "$", trial)
+    if not trial:
+        return True, value
+    if isinstance(value, dict) and "properties" in schema:
+        fixed = {}
+        for key, sub in schema["properties"].items():
+            if key not in value:
+                continue
+            usable, item = _repair(sub, value[key])
+            if usable:
+                fixed[key] = item
+            elif key in schema.get("required", []):
+                return False, None
+        trial = []
+        _check(schema, fixed, "$", trial)
+        return not trial, fixed
+    if isinstance(value, list) and "items" in schema:
+        repaired = [_repair(schema["items"], item) for item in value]
+        return True, [item for usable, item in repaired if usable]
+    candidates = [None, "", "UNKNOWN"]
+    if isinstance(value, str) and "maxLength" in schema:
+        candidates.insert(0, value[:schema["maxLength"]])
+    for candidate in candidates:
+        trial = []
+        _check(schema, candidate, "$", trial)
+        if not trial:
+            return True, candidate
+    return False, None
+
+
+def _isolate(report):
+    """Repair (or, as a last resort, drop) each entry the schema refuses. Returns how many changed."""
+    changed = 0
+    schema = SCHEMA["properties"]
+    usable, database = _repair(schema["database"], report["database"])
+    if database is not report["database"]:
+        changed += 1
+        report["database"] = database if usable else {"updatedAt": EPOCH, "version": "unavailable"}
+    renamed, dropped, targets = {}, set(), []
+    for target in report["targets"]:
+        original = target["id"]
+        if len(original) > 2048:  # keep it unique: a truncated id may not collide with another
+            target = dict(target, id=original[:2031] + "#" + hashlib.sha256(original.encode("utf-8")).hexdigest()[:16])
+            renamed[original] = target["id"]
+        usable, fixed = _repair(schema["targets"]["items"], target)
+        if not usable:
+            changed += 1
+            dropped.add(original)
+            continue
+        if fixed is not target or original in renamed:
+            changed += 1
+        targets.append(fixed)
+    report["targets"] = targets
+    findings = []
+    for finding in report["findings"]:
+        if finding["targetId"] in dropped:
+            changed += 1
+            continue
+        if finding["targetId"] in renamed:
+            finding = dict(finding, targetId=renamed[finding["targetId"]])
+        usable, fixed = _repair(schema["findings"]["items"], finding)
+        if not usable or fixed is not finding:
+            changed += 1
+        if usable:
+            findings.append(fixed)
+    report["findings"] = findings
+    remediation = []
+    for entry in report["remediation"]:
+        usable, fixed = _repair(schema["remediation"]["items"], entry)
+        if not usable or fixed is not entry:
+            changed += 1
+        if usable:
+            remediation.append(fixed)
+    report["remediation"] = remediation
+    return changed
+
+
 def validate(report):
     """Schema and referential errors; an empty list means the report may be served."""
     errors = []

platform/vuln-scanner/src/server.py

@@ -126,7 +126,9 @@ def validate_triage(document):
         if entry["state"] not in REMEDIATION_STATES:
             return [prefix + "unknown state"]
         url = entry["url"]
-        if url is not None and (not isinstance(url, str) or len(url) > 2048 or trivy.safe_url(url) != url):
+        # safe_url is the same rule the report schema enforces, so a value accepted here cannot make
+        # the report invalid. Empty is refused: safe_url returns "" for it, and the schema does not.
+        if url is not None and (not isinstance(url, str) or not url or trivy.safe_url(url) != url):
             return [prefix + "url must be null or an http(s) URL"]
         if parse_ts(entry["updatedAt"]) is None or len(entry["updatedAt"]) > 64:
             return [prefix + "updatedAt must be RFC 3339"]

platform/vuln-scanner/src/state.py

@@ -72,6 +72,9 @@ def new_state():
         "remediation": [],
         "triageHeartbeatAt": None,
         "loadError": None,
+        # The scan key Trivy is running against right now, persisted BEFORE the scan starts: an OOM
+        # kill takes the whole container, so nothing can record the failure afterwards.
+        "inFlight": None,
     }
 
 
@@ -84,6 +87,7 @@ def new_record(now):
         "outcome": None,
         "reason": None,
         "failedSince": None,
+        "killedAt": None,
         "findings": [],
         "firstSeen": {},
     }

platform/vuln-scanner/src/trivy.py

@@ -5,6 +5,8 @@ import re
 import subprocess
 from urllib.parse import urlsplit
 
+from state import parse_ts
+
 log = logging.getLogger("vuln-scanner.trivy")
 
 SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
@@ -13,7 +15,11 @@ _CREDENTIAL = re.compile(
     r"|status(?: code)?:? ?(?:401|403)\b|\b40[13] (?:unauthorized|forbidden)",
     re.IGNORECASE,
 )
-_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
+# Lone surrogates too: json.loads accepts them, and they cannot be encoded as UTF-8 for the page.
+_CONTROL = re.compile(r"[\x00-\x1f\x7f\ud800-\udfff]")
+# Exact and lowercase, matching report.schema.json, whose pattern is case-sensitive: an `HTTPS://x`
+# accepted here would make the schema refuse the entire report.
+_URL_SCHEME = re.compile(r"https?://")
 
 
 class ScanFailure(Exception):
@@ -35,13 +41,18 @@ def text(value, limit=512):
 def safe_url(value):
     """value if it is a plain http(s) URL, else the empty string."""
     url = text(value, 2048)
-    if not url or url != value or re.search(r"\s", url):
+    if not url or url != value or re.search(r"\s", url) or not _URL_SCHEME.match(url):
         return ""
     try:
         parts = urlsplit(url)
     except ValueError:
         return ""
-    return url if parts.scheme.lower() in ("http", "https") and parts.netloc else ""
+    return url if parts.netloc else ""
+
+
+def registry_of(reference):
+    """The lowercase registry host (and port) of a normalised registry/repository reference."""
+    return reference.split("/", 1)[0].lower()
 
 
 def _cvss(vuln):
@@ -95,7 +106,7 @@ def parse_results(document):
 def classify_failure(registry, stderr):
     """Sort a non-zero Trivy exit into credential-required or failed. Reasons name the registry."""
     if _CREDENTIAL.search(stderr or ""):
-        return ScanFailure("credential-required", "registry " + registry + " refused anonymous access")
+        return ScanFailure("credential-required", "registry " + text(registry, 253) + " refused anonymous access")
     lines = [line.strip() for line in (stderr or "").splitlines() if line.strip()]
     return ScanFailure("failed", text(lines[-1], 400) if lines else "trivy exited with an error and no message")
 
@@ -125,9 +136,9 @@ class Runner:
             db = json.loads(result.stdout).get("VulnerabilityDB")
         except (ValueError, AttributeError):
             return None
-        if not isinstance(db, dict) or not db.get("UpdatedAt"):
-            return None
-        return {"updatedAt": db["UpdatedAt"], "version": str(db.get("Version", "unknown"))}
+        if not isinstance(db, dict) or parse_ts(db.get("UpdatedAt")) is None:
+            return None  # an unparseable timestamp is no usable database, and must not reach the report
+        return {"updatedAt": db["UpdatedAt"], "version": text(str(db.get("Version", "unknown")), 128) or "unknown"}
 
     def scan(self, reference):
         args = [
@@ -143,7 +154,7 @@ class Runner:
         if result.returncode < 0:
             raise ScanFailure("failed", "trivy killed by signal " + str(-result.returncode) + " (out of memory?)")
         if result.returncode != 0:
-            raise classify_failure(reference.split("/", 1)[0], result.stderr)
+            raise classify_failure(registry_of(reference), result.stderr)
         try:
             return json.loads(result.stdout)
         except ValueError: