fix(vuln): survive OOM kills, isolate bad upstream strings, label findings per line

Review findings on the integrated scanner:
- An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart
  that image is marked failed and sits out full runs for 20h.
- One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase
  scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build
  repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a
  report is actually published.
- The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names
  it, and Trivy is not invoked.
- loadError clears after a full run against a good inventory.
- Every image under platform/vuln-scanner stays out of Renovate auto-merge.
- The page labels status and target state per line, and mixed acceptance reads as mixed.

The Keycloak client is still an operator action: its key must go in an existing encrypted file
that an agent must not modify. platform/vuln-scanner/README.md has the exact steps.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5.5 committed 2026-11-09 12:28:29 -06:00
1 parent 4c82c64b2f
commit d8b25a67fe
15 files changed
+557 -43

No files matched your search

@@ -16,6 +16,11 @@
# (6) the pod: non-root, read-only rootfs, drop ALL, seccomp RuntimeDefault, no host
# namespaces or hostPath, CPU and memory limits on every container, an ephemeral-storage
# limit on the scanner, and a size-limited /cache emptyDir.
# (8) a renovate.json5 rule covers every file under platform/vuln-scanner: automerge:false,
# minimumReleaseAge of at least 7 days, and no later rule turns auto-merge back on (the
# python image holds the cluster-read token);
# (9) the scanner names the private registry it cannot reach, so that registry reports
# credential-required rather than a permanent failed.
# AND (7) each evaluator is run on a mutated copy and must flag it (non-vacuity).
# =============================================================================
set -euo pipefail
@@ -150,6 +155,33 @@ def eval_renovate(rules):
return bad
SCANNER_FILES = "platform/vuln-scanner/**"
def eval_renovate_files(rules):
positions = [i for i, r in enumerate(rules) if SCANNER_FILES in (r.get("matchFileNames") or [])]
if not positions:
return [f"no packageRule has matchFileNames {SCANNER_FILES}"]
bad = []
rule = rules[positions[-1]]
if rule.get("automerge") is not False:
bad.append("the platform/vuln-scanner file rule is not automerge:false")
age = VERSION_UPDATE_RULE_RE.match(rule.get("minimumReleaseAge") or "")
if not age or int(age.group(1)) < 7:
bad.append("the platform/vuln-scanner file rule has no minimumReleaseAge of at least 7 days")
if any(r.get("automerge") is True for r in rules[positions[-1] + 1:]):
bad.append("a rule after the platform/vuln-scanner file rule re-enables auto-merge")
return bad
def eval_private_registries(pod):
scanner = next(c for c in pod["containers"] if c["name"] == "scanner")
env = {e["name"]: e.get("value") for e in scanner.get("env") or []}
if env.get("VULN_PRIVATE_REGISTRIES") != "gitea.${BASE_DOMAIN}":
return ["the scanner does not name the private registry gitea.${BASE_DOMAIN}"]
return []
def eval_rbac(rbac):
bad = []
croles = [d for d in rbac if d.get("kind") == "ClusterRole"]
@@ -224,6 +256,10 @@ rules = renovate.get("packageRules") or []
check(len(rules) > 5, "(4) parsed renovate.json5 packageRules (parser degenerated)")
for v in eval_renovate(rules):
check(False, f"(4) {v}")
for v in eval_renovate_files(rules):
check(False, f"(8) {v}")
for v in eval_private_registries(pod):
check(False, f"(9) {v}")
rbac = docs(COMP / "rbac.yaml")
for v in eval_rbac(rbac):
@@ -274,6 +310,23 @@ for name, fn in (("Trivy automerge flipped true", ren_auto), ("minimumReleaseAge
("Trivy removed from Renovate rules", ren_drop), ("later auto-merge rule appended", ren_later)):
check(bool(eval_renovate(mutated(fn, rules))), f"(7) Renovate evaluator missed mutant: {name}")
# Renovate file-rule mutants
def files_rule(r):
return next(x for x in r if SCANNER_FILES in (x.get("matchFileNames") or []))
def files_auto(r): files_rule(r)["automerge"] = True
def files_age(r): files_rule(r)["minimumReleaseAge"] = "1 day"
def files_drop(r): r.remove(files_rule(r))
def files_later(r): r.append({"matchDatasources": ["docker"], "automerge": True})
for name, fn in (("file rule automerge true", files_auto), ("file rule age shortened", files_age),
("file rule removed", files_drop), ("later rule re-enables auto-merge", files_later)):
check(bool(eval_renovate_files(mutated(fn, rules))), f"(7) file-rule evaluator missed mutant: {name}")
def reg_drop(p):
next(c for c in p["containers"] if c["name"] == "scanner")["env"][:] = [
e for e in next(c for c in p["containers"] if c["name"] == "scanner")["env"]
if e["name"] != "VULN_PRIVATE_REGISTRIES"]
check(bool(eval_private_registries(mutated(reg_drop, pod))), "(7) private-registry evaluator missed the removal")
# Pod mutants
def pod_root(p): p["securityContext"]["runAsNonRoot"] = False
def pod_host(p): p["hostNetwork"] = True
@@ -94,6 +94,15 @@ def status(when, accepted, errs=()):
return built, built["findings"][0]
def reason_entry(length):
return acceptances.parse_text("- id: CVE-1\n reason: " + "r" * length + "\n expires: 2026-12-01\n", NOW.date())
found, problems = reason_entry(3000)
check("a reason over 2048 characters ignores the entry and reports it", found == [] and any("2048" in p for p in problems))
found, problems = reason_entry(2048)
check("mutation: a reason of exactly 2048 characters is valid", len(found) == 1 and problems == [])
entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)}
built, finding = status(NOW, [entry])
check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted"
@@ -150,6 +150,10 @@ with tempfile.TemporaryDirectory() as tmp:
mutate(document)
status, _, _ = post(internal, document)
check("triage rejects " + name, status == 400)
for name, url in (("an uppercase scheme", "HTTP://gitea.test/pulls/1"), ("an empty url", ""), ("a javascript url", "javascript:alert(1)")):
bad = json.loads(json.dumps(valid))
bad["remediation"][0]["url"] = url
check("triage rejects " + name, post(internal, bad)[0] == 400)
check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415)
status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"})
check("triage rejects an oversized body", status == 413)
@@ -170,6 +170,39 @@ _, _, body = render()
check("default view excludes nothing", all(f["id"] in body for f in REPORT["findings"]))
check("remediation column shows pr-open with a link", 'href="https://gitea.test/pulls/1"' in body and "pr-open" in body)
# Status and state belong to the line they describe (R-VULN-04).
images = [t for t in REPORT["targets"] if t["type"] == "image" and t["workloads"]]
check("fixture has two image targets with workloads", len(images) >= 2)
mixed = copy.deepcopy(REPORT)
first, second = next(t for t in mixed["targets"] if t["id"] == images[0]["id"]), next(t for t in mixed["targets"] if t["id"] == images[1]["id"])
first["state"], first["reason"] = "scanned", None
second["state"], second["reason"] = "stale", "last success long ago"
base = dict(REPORT["findings"][0], id="CVE-MIXED")
base.pop("acceptance", None)
mixed["findings"] = [
dict(base, targetId=first["id"], status="accepted", acceptance={"reason": "only-on-the-first", "expires": "2026-12-01"}),
dict(base, targetId=second["id"], status="fix-available"),
]
_, _, body = render(mixed, "view=cves&q=CVE-MIXED")
check("a mixed CVE reads as mixed, with counts", "mixed: 1 fix-available, 1 accepted" in body)
check("a mixed CVE's row does not carry one image's acceptance", "<td>mixed: 1 fix-available, 1 accepted</td>" in body)
first_line = page.e(first["id"]) + " — target scanned — status accepted (only-on-the-first, until 2026-12-01)"
second_line = page.e(second["id"]) + " — target stale (last success long ago) — status fix-available"
check("the accepted image's line carries its own acceptance", first_line in body)
check("the other image's line carries its own status and state", second_line in body)
check("no line attaches the acceptance to the unaccepted status", "status fix-available (only-on-the-first" not in body)
check("workload lines inside the expansion carry state and status",
"target stale (last success long ago) — status fix-available</li>" in body
and "containers: " + page.e(", ".join(second["workloads"][0]["containers"]) or "none") + " — target stale" in body)
_, _, plain = render(query="view=cves&status=accepted")
check("mutation: a uniformly accepted CVE does not read as mixed", "mixed:" not in plain and "CVE-2026-1003" in plain)
_, _, body = render(mixed, "view=workloads")
check("the workloads images column labels each image's state", page.e(second["id"]) + " — target stale (last success long ago)" in body)
check("the workloads expansion labels each finding's status, image and state",
"status fix-available — " + page.e(second["id"]) + " — target stale (last success long ago)</li>" in body)
check("the workloads expansion keeps an acceptance on its own line",
"status accepted (only-on-the-first, until 2026-12-01) — " + page.e(first["id"]) + " — target scanned</li>" in body)
big = copy.deepcopy(REPORT)
big["findings"] = [dict(REPORT["findings"][0], id="CVE-2026-" + format(i, "05d")) for i in range(250)]
_, _, body = render(big, "view=cves&size=50&page=2")
@@ -95,14 +95,14 @@ class FakeRunner:
pass
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER):
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER, private=()):
path = os.path.join(tmp, "acceptances.yaml")
if not os.path.exists(path):
Path(path).write_text(acc)
clock = clock or Clock()
runner = runner or FakeRunner()
runner.clock = clock
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0, private_registries=private), runner, clock
def targets(eng):
@@ -239,6 +239,184 @@ with tempfile.TemporaryDirectory() as tmp:
thread.join(10)
check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive())
class Killed(BaseException):
"""What an OOM kill looks like from inside the process: no `except Exception` runs."""
class KillingRunner(FakeRunner):
def __init__(self, victim, data_dir):
super().__init__()
self.victim = victim
self.data_dir = data_dir
self.on_disk = None
def scan(self, reference):
if reference == self.victim:
self.on_disk = json.loads(Path(self.data_dir, "state.json").read_text()).get("inFlight")
raise Killed()
return super().scan(reference)
with tempfile.TemporaryDirectory() as tmp:
# R-VULN-05 BLOCKER: a scan that kills the container must cost that image only.
eng, runner, clock = engine(tmp, runner=KillingRunner(POSTGRES, tmp))
killed = False
try:
eng.tick()
except Killed:
killed = True
check("kill: the simulated OOM really aborted the process path", killed)
check("kill: inFlight is on disk BEFORE the scan starts", runner.on_disk == D4)
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
check("kill: startup clears inFlight", eng2.state["inFlight"] is None)
check("kill: clearing is persisted", json.loads(Path(tmp, "state.json").read_text())["inFlight"] is None)
eng2.publish()
t = targets(eng2)
check("kill: the target is failed", t[POSTGRES]["state"] == "failed")
check("kill: the reason is the killed-scan text", t[POSTGRES]["reason"] == main.KILLED_REASON)
eng2.tick()
t = targets(eng2)
check("kill: images after the killed one get scanned", PRIVATE in runner2.scans and t[PRIVATE]["state"] == "credential-required")
check("kill: the killed image is not retried on restart (no wedge)", POSTGRES not in runner2.scans)
check("kill: the killed target stays failed with its reason", t[POSTGRES]["reason"] == main.KILLED_REASON)
check("kill: inFlight is clear after a normal cycle", eng2.state["inFlight"] is None)
clock.advance(hours=21)
eng2.tick()
check("kill: the killed image is retried after the sit-out", runner2.scans.count(POSTGRES) == 1)
check("kill: a successful retry clears the failure", targets(eng2)[POSTGRES]["state"] == "scanned")
check("kill: a successful retry clears killedAt", eng2.state["scans"][D4]["killedAt"] is None)
with tempfile.TemporaryDirectory() as tmp:
eng, runner, clock = engine(tmp)
eng.refresh_inventory()
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
eng2.publish()
check("mutation: a restart with no inFlight fails nothing", targets(eng2)[POSTGRES]["state"] == "pending")
with tempfile.TemporaryDirectory() as tmp:
# A registry the scanner cannot reach is credential-required, and Trivy is never invoked.
eng, runner, clock = engine(tmp, private=[" GHCR.io ", ""])
eng.tick()
t = targets(eng)
check("private registry: reported credential-required", t[PRIVATE]["state"] == "credential-required")
check("private registry: the reason names the registry", "ghcr.io" in t[PRIVATE]["reason"])
check("private registry: Trivy is never invoked for it", PRIVATE not in runner.scans and NIGHTLY not in runner.scans)
check("private registry: other registries still scan", NGINX in runner.scans)
check("private registry: not counted as failed", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0)
with tempfile.TemporaryDirectory() as tmp:
eng, runner, clock = engine(tmp)
eng.tick()
check("mutation: with no private registry configured Trivy is invoked", PRIVATE in runner.scans)
with tempfile.TemporaryDirectory() as tmp:
# A corrupt state.json degrades the run that rebuilds from it, not every run thereafter.
Path(tmp, "state.json").write_text("{not json")
eng, runner, clock = engine(tmp)
check("loadError: set after a corrupt state.json", eng.state["loadError"])
eng.tick()
check("loadError: the rebuilding run is degraded", eng.report["run"]["status"] == "degraded")
check("loadError: cleared after a full run with a good inventory", eng.state["loadError"] is None)
clock.advance(hours=21)
eng.tick()
check("loadError: the next full run is complete", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"])
with tempfile.TemporaryDirectory() as tmp:
Path(tmp, "state.json").write_text("{not json")
eng, runner, clock = engine(tmp)
eng.client.fail = True
eng.tick()
check("mutation: loadError is kept when the inventory failed", eng.state["loadError"])
with tempfile.TemporaryDirectory() as tmp:
# lastCompleteRunAt means a report reached the store; a refused report must not advance it.
eng, runner, clock = engine(tmp)
real_publish = report.publish
report.publish = lambda *_: False
try:
eng.tick()
finally:
report.publish = real_publish
check("a refused publish does not advance lastCompleteRunAt", eng.state["lastCompleteRunAt"] is None)
clock.advance(hours=21)
eng.tick()
check("mutation: a successful publish does advance it", eng.state["lastCompleteRunAt"] is not None)
with tempfile.TemporaryDirectory() as tmp:
# One upstream string costs that string. Each of these used to freeze the whole report.
long_reason = "x" * 3000
eng, runner, clock = engine(tmp, acc="- id: CVE-2026-0010\n reason: " + long_reason + "\n expires: 2026-12-01\n")
runner.outcomes[NGINX] = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
{"VulnerabilityID": "CVE-UP", "Severity": "HIGH", "PkgName": "p", "InstalledVersion": "1",
"PrimaryURL": "HTTPS://upper.example/x", "Title": "t\ud800"}]}]}
eng.tick()
check("hostile strings: the report still publishes", json.loads(Path(tmp, "report.json").read_text())["findings"] and report.validate(eng.report) == [])
up = [f for f in findings(eng, NGINX) if f["id"] == "CVE-UP"][0]
check("hostile strings: an uppercase scheme drops only the URL", up["url"] == "")
check("hostile strings: a lone surrogate is removed", "\ud800" not in up["title"])
check("hostile strings: an over-long acceptance is ignored, the finding stays open", [f for f in findings(eng, POSTGRES) if f["id"] == "CVE-2026-0010"][0]["status"] != "accepted")
check("hostile strings: an over-long acceptance degrades the run", eng.report["run"]["status"] == "degraded")
check("hostile strings: the degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None)
with tempfile.TemporaryDirectory() as tmp:
# Isolation: values already in state, whatever put them there, cannot refuse the whole report.
eng, runner, clock = engine(tmp)
eng.tick()
clean = report.build(eng.state, [], [], clock())
check("mutation: a clean state is not reported as repaired", clean["run"]["status"] == "complete" and clean == report.build(eng.state, [], [], clock()))
record = eng.state["scans"][A1]
record["findings"][0].update(url="HTTP://x", title="t" * 900, severity="weird", cvss=99, package="p" * 900)
eng.state["database"] = {"updatedAt": "not-a-time", "version": "1"}
eng.state["remediation"] = [{"image": "a/b", "state": "pr-open", "url": "HTTP://x", "updatedAt": "2026-11-09T00:00:00Z"}]
target = next(i for i in eng.state["inventory"] if i["id"] == NIGHTLY)
target["id"] = "ghcr.io/" + "o" * 3000 + ":1"
target["image"] = "ghcr.io/" + "o" * 3000
eng.state["scans"][target["id"]] = eng.state["scans"].pop(NIGHTLY) # a by-tag unit is keyed by its id
built = report.build(eng.state, [], [], clock())
check("isolation: the repaired report is schema-valid", report.validate(built) == [])
check("isolation: a repair degrades the run", built["run"]["status"] == "degraded")
bad = [f for f in built["findings"] if f["targetId"] == NGINX and f["title"] == "t" * 512][0]
check("isolation: the bad finding survives with sanitised fields",
bad["url"] == "" and bad["severity"] == "UNKNOWN" and bad["cvss"] is None and len(bad["package"]) == 256)
check("isolation: every other finding is untouched", len(built["findings"]) == len(clean["findings"]))
check("isolation: a bad remediation loses only its url", built["remediation"][0]["url"] is None)
check("isolation: an unusable database is reported as unavailable", built["database"]["version"] == "unavailable")
long_ids = [t for t in built["targets"] if len(t["id"]) > 2000]
check("isolation: an over-long target stays listed", len(long_ids) == 1 and len(long_ids[0]["id"]) <= 2048)
check("isolation: findings follow the shortened target id",
any(f["targetId"] == long_ids[0]["id"] for f in built["findings"]))
class FakeProcess:
returncode = 0
def __init__(self, stdout):
self.stdout = stdout
class VersionRunner(trivy.Runner):
def __init__(self, stdout):
super().__init__("/nonexistent", "/nonexistent")
self.stdout = stdout
def _run(self, args, timeout):
return FakeProcess(self.stdout)
def db_info(updated, version):
return VersionRunner(json.dumps({"VulnerabilityDB": {"UpdatedAt": updated, "Version": version}})).db_info()
check("db_info rejects an unparseable timestamp", db_info("yesterday", "2") is None)
check("db_info rejects a non-string timestamp", db_info(5, "2") is None)
check("db_info bounds the version", len(db_info("2026-11-09T06:13:52.123456789Z", "v" * 500)["version"]) == 128)
check("mutation: db_info keeps a good database", db_info("2026-11-09T06:13:52Z", "2") == {"updatedAt": "2026-11-09T06:13:52Z", "version": "2"})
check("a credential failure's reason is bounded", len(trivy.classify_failure("r" * 3000, "401 unauthorized").reason) <= 1024)
for url, want in (("https://a.test/x", "https://a.test/x"), ("http://a.test/", "http://a.test/"), ("HTTPS://a.test/x", ""),
("HTTP://a.test/x", ""), ("Https://a.test/x", ""), ("javascript:alert(1)", ""), ("", "")):
check("safe_url " + repr(url), trivy.safe_url(url) == want)
check("every url safe_url keeps satisfies the report schema",
report.re.search(report.SCHEMA["properties"]["findings"]["items"]["properties"]["url"]["pattern"], trivy.safe_url("https://a.test/x")))
# Recorded Trivy output through the normaliser: untrusted text.
parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
{"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y",