fix(vuln): survive OOM kills, isolate bad upstream strings, label findings per line
Review findings on the integrated scanner: - An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart that image is marked failed and sits out full runs for 20h. - One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a report is actually published. - The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names it, and Trivy is not invoked. - loadError clears after a full run against a good inventory. - Every image under platform/vuln-scanner stays out of Renovate auto-merge. - The page labels status and target state per line, and mixed acceptance reads as mixed. The Keycloak client is still an operator action: its key must go in an existing encrypted file that an agent must not modify. platform/vuln-scanner/README.md has the exact steps. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed
1 parent
4c82c64b2f
commit
d8b25a67fe
15 files changed
+557
-43
No files matched your search
@@ -95,14 +95,14 @@ class FakeRunner:
|
||||
pass
|
||||
|
||||
|
||||
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER):
|
||||
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER, private=()):
|
||||
path = os.path.join(tmp, "acceptances.yaml")
|
||||
if not os.path.exists(path):
|
||||
Path(path).write_text(acc)
|
||||
clock = clock or Clock()
|
||||
runner = runner or FakeRunner()
|
||||
runner.clock = clock
|
||||
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock
|
||||
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0, private_registries=private), runner, clock
|
||||
|
||||
|
||||
def targets(eng):
|
||||
@@ -239,6 +239,184 @@ with tempfile.TemporaryDirectory() as tmp:
|
||||
thread.join(10)
|
||||
check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive())
|
||||
|
||||
class Killed(BaseException):
|
||||
"""What an OOM kill looks like from inside the process: no `except Exception` runs."""
|
||||
|
||||
|
||||
class KillingRunner(FakeRunner):
|
||||
def __init__(self, victim, data_dir):
|
||||
super().__init__()
|
||||
self.victim = victim
|
||||
self.data_dir = data_dir
|
||||
self.on_disk = None
|
||||
|
||||
def scan(self, reference):
|
||||
if reference == self.victim:
|
||||
self.on_disk = json.loads(Path(self.data_dir, "state.json").read_text()).get("inFlight")
|
||||
raise Killed()
|
||||
return super().scan(reference)
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# R-VULN-05 BLOCKER: a scan that kills the container must cost that image only.
|
||||
eng, runner, clock = engine(tmp, runner=KillingRunner(POSTGRES, tmp))
|
||||
killed = False
|
||||
try:
|
||||
eng.tick()
|
||||
except Killed:
|
||||
killed = True
|
||||
check("kill: the simulated OOM really aborted the process path", killed)
|
||||
check("kill: inFlight is on disk BEFORE the scan starts", runner.on_disk == D4)
|
||||
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
|
||||
check("kill: startup clears inFlight", eng2.state["inFlight"] is None)
|
||||
check("kill: clearing is persisted", json.loads(Path(tmp, "state.json").read_text())["inFlight"] is None)
|
||||
eng2.publish()
|
||||
t = targets(eng2)
|
||||
check("kill: the target is failed", t[POSTGRES]["state"] == "failed")
|
||||
check("kill: the reason is the killed-scan text", t[POSTGRES]["reason"] == main.KILLED_REASON)
|
||||
eng2.tick()
|
||||
t = targets(eng2)
|
||||
check("kill: images after the killed one get scanned", PRIVATE in runner2.scans and t[PRIVATE]["state"] == "credential-required")
|
||||
check("kill: the killed image is not retried on restart (no wedge)", POSTGRES not in runner2.scans)
|
||||
check("kill: the killed target stays failed with its reason", t[POSTGRES]["reason"] == main.KILLED_REASON)
|
||||
check("kill: inFlight is clear after a normal cycle", eng2.state["inFlight"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng2.tick()
|
||||
check("kill: the killed image is retried after the sit-out", runner2.scans.count(POSTGRES) == 1)
|
||||
check("kill: a successful retry clears the failure", targets(eng2)[POSTGRES]["state"] == "scanned")
|
||||
check("kill: a successful retry clears killedAt", eng2.state["scans"][D4]["killedAt"] is None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.refresh_inventory()
|
||||
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
|
||||
eng2.publish()
|
||||
check("mutation: a restart with no inFlight fails nothing", targets(eng2)[POSTGRES]["state"] == "pending")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A registry the scanner cannot reach is credential-required, and Trivy is never invoked.
|
||||
eng, runner, clock = engine(tmp, private=[" GHCR.io ", ""])
|
||||
eng.tick()
|
||||
t = targets(eng)
|
||||
check("private registry: reported credential-required", t[PRIVATE]["state"] == "credential-required")
|
||||
check("private registry: the reason names the registry", "ghcr.io" in t[PRIVATE]["reason"])
|
||||
check("private registry: Trivy is never invoked for it", PRIVATE not in runner.scans and NIGHTLY not in runner.scans)
|
||||
check("private registry: other registries still scan", NGINX in runner.scans)
|
||||
check("private registry: not counted as failed", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
check("mutation: with no private registry configured Trivy is invoked", PRIVATE in runner.scans)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A corrupt state.json degrades the run that rebuilds from it, not every run thereafter.
|
||||
Path(tmp, "state.json").write_text("{not json")
|
||||
eng, runner, clock = engine(tmp)
|
||||
check("loadError: set after a corrupt state.json", eng.state["loadError"])
|
||||
eng.tick()
|
||||
check("loadError: the rebuilding run is degraded", eng.report["run"]["status"] == "degraded")
|
||||
check("loadError: cleared after a full run with a good inventory", eng.state["loadError"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng.tick()
|
||||
check("loadError: the next full run is complete", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
Path(tmp, "state.json").write_text("{not json")
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.client.fail = True
|
||||
eng.tick()
|
||||
check("mutation: loadError is kept when the inventory failed", eng.state["loadError"])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# lastCompleteRunAt means a report reached the store; a refused report must not advance it.
|
||||
eng, runner, clock = engine(tmp)
|
||||
real_publish = report.publish
|
||||
report.publish = lambda *_: False
|
||||
try:
|
||||
eng.tick()
|
||||
finally:
|
||||
report.publish = real_publish
|
||||
check("a refused publish does not advance lastCompleteRunAt", eng.state["lastCompleteRunAt"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng.tick()
|
||||
check("mutation: a successful publish does advance it", eng.state["lastCompleteRunAt"] is not None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# One upstream string costs that string. Each of these used to freeze the whole report.
|
||||
long_reason = "x" * 3000
|
||||
eng, runner, clock = engine(tmp, acc="- id: CVE-2026-0010\n reason: " + long_reason + "\n expires: 2026-12-01\n")
|
||||
runner.outcomes[NGINX] = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-UP", "Severity": "HIGH", "PkgName": "p", "InstalledVersion": "1",
|
||||
"PrimaryURL": "HTTPS://upper.example/x", "Title": "t\ud800"}]}]}
|
||||
eng.tick()
|
||||
check("hostile strings: the report still publishes", json.loads(Path(tmp, "report.json").read_text())["findings"] and report.validate(eng.report) == [])
|
||||
up = [f for f in findings(eng, NGINX) if f["id"] == "CVE-UP"][0]
|
||||
check("hostile strings: an uppercase scheme drops only the URL", up["url"] == "")
|
||||
check("hostile strings: a lone surrogate is removed", "\ud800" not in up["title"])
|
||||
check("hostile strings: an over-long acceptance is ignored, the finding stays open", [f for f in findings(eng, POSTGRES) if f["id"] == "CVE-2026-0010"][0]["status"] != "accepted")
|
||||
check("hostile strings: an over-long acceptance degrades the run", eng.report["run"]["status"] == "degraded")
|
||||
check("hostile strings: the degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Isolation: values already in state, whatever put them there, cannot refuse the whole report.
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
clean = report.build(eng.state, [], [], clock())
|
||||
check("mutation: a clean state is not reported as repaired", clean["run"]["status"] == "complete" and clean == report.build(eng.state, [], [], clock()))
|
||||
record = eng.state["scans"][A1]
|
||||
record["findings"][0].update(url="HTTP://x", title="t" * 900, severity="weird", cvss=99, package="p" * 900)
|
||||
eng.state["database"] = {"updatedAt": "not-a-time", "version": "1"}
|
||||
eng.state["remediation"] = [{"image": "a/b", "state": "pr-open", "url": "HTTP://x", "updatedAt": "2026-11-09T00:00:00Z"}]
|
||||
target = next(i for i in eng.state["inventory"] if i["id"] == NIGHTLY)
|
||||
target["id"] = "ghcr.io/" + "o" * 3000 + ":1"
|
||||
target["image"] = "ghcr.io/" + "o" * 3000
|
||||
eng.state["scans"][target["id"]] = eng.state["scans"].pop(NIGHTLY) # a by-tag unit is keyed by its id
|
||||
built = report.build(eng.state, [], [], clock())
|
||||
check("isolation: the repaired report is schema-valid", report.validate(built) == [])
|
||||
check("isolation: a repair degrades the run", built["run"]["status"] == "degraded")
|
||||
bad = [f for f in built["findings"] if f["targetId"] == NGINX and f["title"] == "t" * 512][0]
|
||||
check("isolation: the bad finding survives with sanitised fields",
|
||||
bad["url"] == "" and bad["severity"] == "UNKNOWN" and bad["cvss"] is None and len(bad["package"]) == 256)
|
||||
check("isolation: every other finding is untouched", len(built["findings"]) == len(clean["findings"]))
|
||||
check("isolation: a bad remediation loses only its url", built["remediation"][0]["url"] is None)
|
||||
check("isolation: an unusable database is reported as unavailable", built["database"]["version"] == "unavailable")
|
||||
long_ids = [t for t in built["targets"] if len(t["id"]) > 2000]
|
||||
check("isolation: an over-long target stays listed", len(long_ids) == 1 and len(long_ids[0]["id"]) <= 2048)
|
||||
check("isolation: findings follow the shortened target id",
|
||||
any(f["targetId"] == long_ids[0]["id"] for f in built["findings"]))
|
||||
|
||||
|
||||
class FakeProcess:
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, stdout):
|
||||
self.stdout = stdout
|
||||
|
||||
|
||||
class VersionRunner(trivy.Runner):
|
||||
def __init__(self, stdout):
|
||||
super().__init__("/nonexistent", "/nonexistent")
|
||||
self.stdout = stdout
|
||||
|
||||
def _run(self, args, timeout):
|
||||
return FakeProcess(self.stdout)
|
||||
|
||||
|
||||
def db_info(updated, version):
|
||||
return VersionRunner(json.dumps({"VulnerabilityDB": {"UpdatedAt": updated, "Version": version}})).db_info()
|
||||
|
||||
|
||||
check("db_info rejects an unparseable timestamp", db_info("yesterday", "2") is None)
|
||||
check("db_info rejects a non-string timestamp", db_info(5, "2") is None)
|
||||
check("db_info bounds the version", len(db_info("2026-11-09T06:13:52.123456789Z", "v" * 500)["version"]) == 128)
|
||||
check("mutation: db_info keeps a good database", db_info("2026-11-09T06:13:52Z", "2") == {"updatedAt": "2026-11-09T06:13:52Z", "version": "2"})
|
||||
check("a credential failure's reason is bounded", len(trivy.classify_failure("r" * 3000, "401 unauthorized").reason) <= 1024)
|
||||
|
||||
for url, want in (("https://a.test/x", "https://a.test/x"), ("http://a.test/", "http://a.test/"), ("HTTPS://a.test/x", ""),
|
||||
("HTTP://a.test/x", ""), ("Https://a.test/x", ""), ("javascript:alert(1)", ""), ("", "")):
|
||||
check("safe_url " + repr(url), trivy.safe_url(url) == want)
|
||||
check("every url safe_url keeps satisfies the report schema",
|
||||
report.re.search(report.SCHEMA["properties"]["findings"]["items"]["properties"]["url"]["pattern"], trivy.safe_url("https://a.test/x")))
|
||||
|
||||
# Recorded Trivy output through the normaliser: untrusted text.
|
||||
parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y",
|
||||
|
||||
Reference in new issue
Block a user