fix(vuln): survive OOM kills, isolate bad upstream strings, label findings per line
Review findings on the integrated scanner: - An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart that image is marked failed and sits out full runs for 20h. - One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a report is actually published. - The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names it, and Trivy is not invoked. - loadError clears after a full run against a good inventory. - Every image under platform/vuln-scanner stays out of Renovate auto-merge. - The page labels status and target state per line, and mixed acceptance reads as mixed. The Keycloak client is still an operator action: its key must go in an existing encrypted file that an agent must not modify. platform/vuln-scanner/README.md has the exact steps. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
4c82c64b2f
commit
d8b25a67fe
15 files changed
+557
-43
No files matched your search
@@ -23,15 +23,21 @@ log = logging.getLogger("vuln-scanner")
|
||||
FULL_RUN_INTERVAL = timedelta(hours=20)
|
||||
DB_MAX_AGE = timedelta(hours=48)
|
||||
DB_REFRESH_AGE = timedelta(hours=24)
|
||||
KILLED_REASON = "scanner killed during scan (OOM/eviction?)"
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120):
|
||||
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120,
|
||||
private_registries=()):
|
||||
self.store = st.Store(data_dir)
|
||||
self.data_dir = data_dir
|
||||
self.client = client
|
||||
self.runner = runner
|
||||
self.acceptances_path = acceptances_path
|
||||
# Registries known to need credentials the scanner deliberately does not hold (R-VULN-09) and,
|
||||
# being on the LAN, not reachable from its egress policy: a scan would end in a connection
|
||||
# error (`failed`) rather than a 401 (`credential-required`), so it is never attempted.
|
||||
self.private_registries = {name.strip().lower() for name in private_registries if name.strip()}
|
||||
self.clock = clock
|
||||
self.tick_seconds = tick_seconds
|
||||
self.publish_seconds = publish_seconds
|
||||
@@ -43,6 +49,18 @@ class Engine:
|
||||
self.loop_thread = None
|
||||
self._last_publish = 0.0
|
||||
self._last_inventory = None
|
||||
self._recover_interrupted_scan()
|
||||
|
||||
def _recover_interrupted_scan(self):
|
||||
"""An inFlight key at startup means the last process died inside that scan."""
|
||||
key = self.state.get("inFlight")
|
||||
if not key:
|
||||
return
|
||||
log.error("the previous process was killed while scanning %s; marking it failed", key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is not None:
|
||||
record["killedAt"] = fmt_ts(self.clock())
|
||||
self._record_failure(key, "failed", KILLED_REASON)
|
||||
|
||||
# --- views used by the HTTP servers; none of them waits for a scan --------------------------
|
||||
def report_bytes(self):
|
||||
@@ -67,16 +85,19 @@ class Engine:
|
||||
|
||||
# --- persistence -----------------------------------------------------------------------------
|
||||
def publish(self):
|
||||
"""True when the report was written; False when it was refused and the previous one stands."""
|
||||
with self.lock:
|
||||
now = self.clock()
|
||||
entries, errors = acceptances.load(self.acceptances_path, now.date())
|
||||
for problem in errors:
|
||||
log.error("invalid acceptance ignored: %s", problem)
|
||||
built = report_mod.build(self.state, entries, errors, now)
|
||||
if report_mod.publish(built, self.data_dir):
|
||||
published = report_mod.publish(built, self.data_dir)
|
||||
if published:
|
||||
self.report = built
|
||||
self.report_json = report_mod.serialise(built)
|
||||
self._last_publish = time.monotonic()
|
||||
return published
|
||||
|
||||
# --- inventory -------------------------------------------------------------------------------
|
||||
def refresh_inventory(self):
|
||||
@@ -152,6 +173,14 @@ class Engine:
|
||||
return timedelta.max if moment is None else self.clock() - moment
|
||||
|
||||
def _scan_unit(self, key, reference):
|
||||
registry = trivy.registry_of(reference)
|
||||
if registry in self.private_registries or registry.split(":", 1)[0] in self.private_registries:
|
||||
self._record_failure(key, "credential-required",
|
||||
"registry " + registry + " is private and the scanner holds no credential for it")
|
||||
return
|
||||
with self.lock:
|
||||
self.state["inFlight"] = key
|
||||
self.store.save(self.state)
|
||||
try:
|
||||
found = trivy.parse_results(self.runner.scan(reference))
|
||||
except trivy.ScanFailure as exc:
|
||||
@@ -166,11 +195,24 @@ class Engine:
|
||||
except Exception: # pylint: disable=broad-except
|
||||
log.exception("scan cache cleanup failed")
|
||||
|
||||
def _finish_in_flight(self, key):
|
||||
"""Cleared in the same write as the scan's outcome, so no crash can separate the two."""
|
||||
if self.state.get("inFlight") == key:
|
||||
self.state["inFlight"] = None
|
||||
|
||||
def _killed_recently(self, key):
|
||||
"""A unit that killed the scanner sits out full runs for a day: retrying it first on every
|
||||
restart would OOM again before any later image, which is the wedge this exists to prevent."""
|
||||
killed = parse_ts(self.state["scans"][key].get("killedAt"))
|
||||
return killed is not None and self.clock() - killed < FULL_RUN_INTERVAL
|
||||
|
||||
def _record_failure(self, key, kind, reason):
|
||||
now = fmt_ts(self.clock())
|
||||
with self.lock:
|
||||
self._finish_in_flight(key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is None:
|
||||
self.store.save(self.state)
|
||||
return
|
||||
record["lastAttemptAt"] = now
|
||||
record["outcome"] = kind
|
||||
@@ -181,10 +223,13 @@ class Engine:
|
||||
def _record_success(self, key, found):
|
||||
now = fmt_ts(self.clock())
|
||||
with self.lock:
|
||||
self._finish_in_flight(key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is None:
|
||||
self.store.save(self.state)
|
||||
return
|
||||
record["lastAttemptAt"] = now
|
||||
record["killedAt"] = None
|
||||
record["lastSuccessAt"] = now
|
||||
record["outcome"] = "ok"
|
||||
record["reason"] = None
|
||||
@@ -215,7 +260,10 @@ class Engine:
|
||||
started = self.clock()
|
||||
log.info("full run starting")
|
||||
database_ok = self._refresh_database(always=True)
|
||||
self._scan_queue(sorted(self._units()))
|
||||
self._scan_queue([key for key in sorted(self._units()) if not self._killed_recently(key)])
|
||||
if self.stop.is_set():
|
||||
log.info("full run interrupted by shutdown; not recorded")
|
||||
return
|
||||
info = self.state["database"]
|
||||
now = self.clock()
|
||||
degraded = not database_ok or info is None or self._db_age(info) > DB_MAX_AGE
|
||||
@@ -225,8 +273,15 @@ class Engine:
|
||||
status = "degraded" if degraded else "complete"
|
||||
with self.lock:
|
||||
self.state["run"] = {"startedAt": fmt_ts(started), "finishedAt": fmt_ts(now), "status": status}
|
||||
if not degraded:
|
||||
self.store.save(self.state)
|
||||
# Only a report that actually reached the store counts as a completed run. Otherwise a
|
||||
# refused report would freeze the page while VulnScanStale stayed quiet.
|
||||
if self.publish() and not degraded and self.report["run"]["status"] == "complete":
|
||||
self.state["lastCompleteRunAt"] = fmt_ts(now)
|
||||
# A corrupt state.json degrades the run that rebuilds from it, and no further: once a full
|
||||
# run has completed against a good inventory, the state is whole again.
|
||||
if self.state.get("loadError") and self._last_inventory is not None and not self.state.get("inventoryError"):
|
||||
self.state["loadError"] = None
|
||||
self.store.save(self.state)
|
||||
log.info("full run finished: %s", status)
|
||||
|
||||
@@ -262,7 +317,9 @@ def main():
|
||||
tools_dir = os.environ.get("VULN_TOOLS_DIR", "/tools")
|
||||
trivy_bin = os.environ.get("TRIVY_BIN", os.path.join(tools_dir, "trivy"))
|
||||
runner = trivy.Runner(trivy_bin, cache_dir, int(os.environ.get("VULN_SCAN_TIMEOUT_SECONDS", "600")))
|
||||
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"))
|
||||
private_registries = os.environ.get("VULN_PRIVATE_REGISTRIES", "").split(",")
|
||||
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"),
|
||||
private_registries=private_registries)
|
||||
server.start_all(engine, ("127.0.0.1", 8080), ("0.0.0.0", 8081), ("0.0.0.0", 9090))
|
||||
finished = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: finished.set())
|
||||
|
||||
Reference in new issue
Block a user