From d8b25a67feeea46f3e1311d4e8fb15aedaaad8e8 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 9 Nov 2026 12:28:29 -0600 Subject: [PATCH] fix(vuln): survive OOM kills, isolate bad upstream strings, label findings per line Review findings on the integrated scanner: - An OOM kill no longer wedges the loop: inFlight is persisted before each scan, and on restart that image is marked failed and sits out full runs for 20h. - One bad upstream string can no longer freeze the report: safe_url requires an exact lowercase scheme, acceptance reasons are bounded, /triage urls use the same rule, and report.build repairs or drops any entry the schema would refuse. lastCompleteRunAt advances only after a report is actually published. - The private registry reports credential-required, not failed: VULN_PRIVATE_REGISTRIES names it, and Trivy is not invoked. - loadError clears after a full run against a good inventory. - Every image under platform/vuln-scanner stays out of Renovate auto-merge. - The page labels status and target state per line, and mixed acceptance reads as mixed. The Keycloak client is still an operator action: its key must go in an existing encrypted file that an agent must not modify. platform/vuln-scanner/README.md has the exact steps. Co-Authored-By: Claude Opus 5.5 --- .../vuln-scanner-supply-chain/check.sh | 53 +++++ .../vuln-scanner-acceptances/check.py | 9 + .../vuln-scanner-api-metrics/check.py | 4 + .../l1-rendering/vuln-scanner-page/check.py | 33 ++++ .../vuln-scanner-scan-loop/check.py | 182 +++++++++++++++++- platform/vuln-scanner/README.md | 59 ++++-- platform/vuln-scanner/deployment.yaml | 8 +- platform/vuln-scanner/src/acceptances.py | 3 + platform/vuln-scanner/src/main.py | 67 ++++++- platform/vuln-scanner/src/page.py | 47 +++-- platform/vuln-scanner/src/report.py | 90 ++++++++- platform/vuln-scanner/src/server.py | 4 +- platform/vuln-scanner/src/state.py | 4 + platform/vuln-scanner/src/trivy.py | 27 ++- renovate.json5 | 10 + 15 files changed, 557 insertions(+), 43 deletions(-) diff --git a/conformance/tests/l0-static/vuln-scanner-supply-chain/check.sh b/conformance/tests/l0-static/vuln-scanner-supply-chain/check.sh index 56320e8..e398522 100755 --- a/conformance/tests/l0-static/vuln-scanner-supply-chain/check.sh +++ b/conformance/tests/l0-static/vuln-scanner-supply-chain/check.sh @@ -16,6 +16,11 @@ # (6) the pod: non-root, read-only rootfs, drop ALL, seccomp RuntimeDefault, no host # namespaces or hostPath, CPU and memory limits on every container, an ephemeral-storage # limit on the scanner, and a size-limited /cache emptyDir. +# (8) a renovate.json5 rule covers every file under platform/vuln-scanner: automerge:false, +# minimumReleaseAge of at least 7 days, and no later rule turns auto-merge back on (the +# python image holds the cluster-read token); +# (9) the scanner names the private registry it cannot reach, so that registry reports +# credential-required rather than a permanent failed. # AND (7) each evaluator is run on a mutated copy and must flag it (non-vacuity). # ============================================================================= set -euo pipefail @@ -150,6 +155,33 @@ def eval_renovate(rules): return bad +SCANNER_FILES = "platform/vuln-scanner/**" + + +def eval_renovate_files(rules): + positions = [i for i, r in enumerate(rules) if SCANNER_FILES in (r.get("matchFileNames") or [])] + if not positions: + return [f"no packageRule has matchFileNames {SCANNER_FILES}"] + bad = [] + rule = rules[positions[-1]] + if rule.get("automerge") is not False: + bad.append("the platform/vuln-scanner file rule is not automerge:false") + age = VERSION_UPDATE_RULE_RE.match(rule.get("minimumReleaseAge") or "") + if not age or int(age.group(1)) < 7: + bad.append("the platform/vuln-scanner file rule has no minimumReleaseAge of at least 7 days") + if any(r.get("automerge") is True for r in rules[positions[-1] + 1:]): + bad.append("a rule after the platform/vuln-scanner file rule re-enables auto-merge") + return bad + + +def eval_private_registries(pod): + scanner = next(c for c in pod["containers"] if c["name"] == "scanner") + env = {e["name"]: e.get("value") for e in scanner.get("env") or []} + if env.get("VULN_PRIVATE_REGISTRIES") != "gitea.${BASE_DOMAIN}": + return ["the scanner does not name the private registry gitea.${BASE_DOMAIN}"] + return [] + + def eval_rbac(rbac): bad = [] croles = [d for d in rbac if d.get("kind") == "ClusterRole"] @@ -224,6 +256,10 @@ rules = renovate.get("packageRules") or [] check(len(rules) > 5, "(4) parsed renovate.json5 packageRules (parser degenerated)") for v in eval_renovate(rules): check(False, f"(4) {v}") +for v in eval_renovate_files(rules): + check(False, f"(8) {v}") +for v in eval_private_registries(pod): + check(False, f"(9) {v}") rbac = docs(COMP / "rbac.yaml") for v in eval_rbac(rbac): @@ -274,6 +310,23 @@ for name, fn in (("Trivy automerge flipped true", ren_auto), ("minimumReleaseAge ("Trivy removed from Renovate rules", ren_drop), ("later auto-merge rule appended", ren_later)): check(bool(eval_renovate(mutated(fn, rules))), f"(7) Renovate evaluator missed mutant: {name}") +# Renovate file-rule mutants +def files_rule(r): + return next(x for x in r if SCANNER_FILES in (x.get("matchFileNames") or [])) +def files_auto(r): files_rule(r)["automerge"] = True +def files_age(r): files_rule(r)["minimumReleaseAge"] = "1 day" +def files_drop(r): r.remove(files_rule(r)) +def files_later(r): r.append({"matchDatasources": ["docker"], "automerge": True}) +for name, fn in (("file rule automerge true", files_auto), ("file rule age shortened", files_age), + ("file rule removed", files_drop), ("later rule re-enables auto-merge", files_later)): + check(bool(eval_renovate_files(mutated(fn, rules))), f"(7) file-rule evaluator missed mutant: {name}") + +def reg_drop(p): + next(c for c in p["containers"] if c["name"] == "scanner")["env"][:] = [ + e for e in next(c for c in p["containers"] if c["name"] == "scanner")["env"] + if e["name"] != "VULN_PRIVATE_REGISTRIES"] +check(bool(eval_private_registries(mutated(reg_drop, pod))), "(7) private-registry evaluator missed the removal") + # Pod mutants def pod_root(p): p["securityContext"]["runAsNonRoot"] = False def pod_host(p): p["hostNetwork"] = True diff --git a/conformance/tests/l1-rendering/vuln-scanner-acceptances/check.py b/conformance/tests/l1-rendering/vuln-scanner-acceptances/check.py index 6fb3560..8cb4d2f 100644 --- a/conformance/tests/l1-rendering/vuln-scanner-acceptances/check.py +++ b/conformance/tests/l1-rendering/vuln-scanner-acceptances/check.py @@ -94,6 +94,15 @@ def status(when, accepted, errs=()): return built, built["findings"][0] +def reason_entry(length): + return acceptances.parse_text("- id: CVE-1\n reason: " + "r" * length + "\n expires: 2026-12-01\n", NOW.date()) + + +found, problems = reason_entry(3000) +check("a reason over 2048 characters ignores the entry and reports it", found == [] and any("2048" in p for p in problems)) +found, problems = reason_entry(2048) +check("mutation: a reason of exactly 2048 characters is valid", len(found) == 1 and problems == []) + entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)} built, finding = status(NOW, [entry]) check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted" diff --git a/conformance/tests/l1-rendering/vuln-scanner-api-metrics/check.py b/conformance/tests/l1-rendering/vuln-scanner-api-metrics/check.py index 4b81880..4a43f05 100644 --- a/conformance/tests/l1-rendering/vuln-scanner-api-metrics/check.py +++ b/conformance/tests/l1-rendering/vuln-scanner-api-metrics/check.py @@ -150,6 +150,10 @@ with tempfile.TemporaryDirectory() as tmp: mutate(document) status, _, _ = post(internal, document) check("triage rejects " + name, status == 400) + for name, url in (("an uppercase scheme", "HTTP://gitea.test/pulls/1"), ("an empty url", ""), ("a javascript url", "javascript:alert(1)")): + bad = json.loads(json.dumps(valid)) + bad["remediation"][0]["url"] = url + check("triage rejects " + name, post(internal, bad)[0] == 400) check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415) status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"}) check("triage rejects an oversized body", status == 413) diff --git a/conformance/tests/l1-rendering/vuln-scanner-page/check.py b/conformance/tests/l1-rendering/vuln-scanner-page/check.py index 3a41da1..a8e4a39 100644 --- a/conformance/tests/l1-rendering/vuln-scanner-page/check.py +++ b/conformance/tests/l1-rendering/vuln-scanner-page/check.py @@ -170,6 +170,39 @@ _, _, body = render() check("default view excludes nothing", all(f["id"] in body for f in REPORT["findings"])) check("remediation column shows pr-open with a link", 'href="https://gitea.test/pulls/1"' in body and "pr-open" in body) +# Status and state belong to the line they describe (R-VULN-04). +images = [t for t in REPORT["targets"] if t["type"] == "image" and t["workloads"]] +check("fixture has two image targets with workloads", len(images) >= 2) +mixed = copy.deepcopy(REPORT) +first, second = next(t for t in mixed["targets"] if t["id"] == images[0]["id"]), next(t for t in mixed["targets"] if t["id"] == images[1]["id"]) +first["state"], first["reason"] = "scanned", None +second["state"], second["reason"] = "stale", "last success long ago" +base = dict(REPORT["findings"][0], id="CVE-MIXED") +base.pop("acceptance", None) +mixed["findings"] = [ + dict(base, targetId=first["id"], status="accepted", acceptance={"reason": "only-on-the-first", "expires": "2026-12-01"}), + dict(base, targetId=second["id"], status="fix-available"), +] +_, _, body = render(mixed, "view=cves&q=CVE-MIXED") +check("a mixed CVE reads as mixed, with counts", "mixed: 1 fix-available, 1 accepted" in body) +check("a mixed CVE's row does not carry one image's acceptance", "mixed: 1 fix-available, 1 accepted" in body) +first_line = page.e(first["id"]) + " — target scanned — status accepted (only-on-the-first, until 2026-12-01)" +second_line = page.e(second["id"]) + " — target stale (last success long ago) — status fix-available" +check("the accepted image's line carries its own acceptance", first_line in body) +check("the other image's line carries its own status and state", second_line in body) +check("no line attaches the acceptance to the unaccepted status", "status fix-available (only-on-the-first" not in body) +check("workload lines inside the expansion carry state and status", + "target stale (last success long ago) — status fix-available" in body + and "containers: " + page.e(", ".join(second["workloads"][0]["containers"]) or "none") + " — target stale" in body) +_, _, plain = render(query="view=cves&status=accepted") +check("mutation: a uniformly accepted CVE does not read as mixed", "mixed:" not in plain and "CVE-2026-1003" in plain) +_, _, body = render(mixed, "view=workloads") +check("the workloads images column labels each image's state", page.e(second["id"]) + " — target stale (last success long ago)" in body) +check("the workloads expansion labels each finding's status, image and state", + "status fix-available — " + page.e(second["id"]) + " — target stale (last success long ago)" in body) +check("the workloads expansion keeps an acceptance on its own line", + "status accepted (only-on-the-first, until 2026-12-01) — " + page.e(first["id"]) + " — target scanned" in body) + big = copy.deepcopy(REPORT) big["findings"] = [dict(REPORT["findings"][0], id="CVE-2026-" + format(i, "05d")) for i in range(250)] _, _, body = render(big, "view=cves&size=50&page=2") diff --git a/conformance/tests/l1-rendering/vuln-scanner-scan-loop/check.py b/conformance/tests/l1-rendering/vuln-scanner-scan-loop/check.py index f86ea2d..391304f 100644 --- a/conformance/tests/l1-rendering/vuln-scanner-scan-loop/check.py +++ b/conformance/tests/l1-rendering/vuln-scanner-scan-loop/check.py @@ -95,14 +95,14 @@ class FakeRunner: pass -def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER): +def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER, private=()): path = os.path.join(tmp, "acceptances.yaml") if not os.path.exists(path): Path(path).write_text(acc) clock = clock or Clock() runner = runner or FakeRunner() runner.clock = clock - return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock + return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0, private_registries=private), runner, clock def targets(eng): @@ -239,6 +239,184 @@ with tempfile.TemporaryDirectory() as tmp: thread.join(10) check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive()) +class Killed(BaseException): + """What an OOM kill looks like from inside the process: no `except Exception` runs.""" + + +class KillingRunner(FakeRunner): + def __init__(self, victim, data_dir): + super().__init__() + self.victim = victim + self.data_dir = data_dir + self.on_disk = None + + def scan(self, reference): + if reference == self.victim: + self.on_disk = json.loads(Path(self.data_dir, "state.json").read_text()).get("inFlight") + raise Killed() + return super().scan(reference) + + +with tempfile.TemporaryDirectory() as tmp: + # R-VULN-05 BLOCKER: a scan that kills the container must cost that image only. + eng, runner, clock = engine(tmp, runner=KillingRunner(POSTGRES, tmp)) + killed = False + try: + eng.tick() + except Killed: + killed = True + check("kill: the simulated OOM really aborted the process path", killed) + check("kill: inFlight is on disk BEFORE the scan starts", runner.on_disk == D4) + eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock) + check("kill: startup clears inFlight", eng2.state["inFlight"] is None) + check("kill: clearing is persisted", json.loads(Path(tmp, "state.json").read_text())["inFlight"] is None) + eng2.publish() + t = targets(eng2) + check("kill: the target is failed", t[POSTGRES]["state"] == "failed") + check("kill: the reason is the killed-scan text", t[POSTGRES]["reason"] == main.KILLED_REASON) + eng2.tick() + t = targets(eng2) + check("kill: images after the killed one get scanned", PRIVATE in runner2.scans and t[PRIVATE]["state"] == "credential-required") + check("kill: the killed image is not retried on restart (no wedge)", POSTGRES not in runner2.scans) + check("kill: the killed target stays failed with its reason", t[POSTGRES]["reason"] == main.KILLED_REASON) + check("kill: inFlight is clear after a normal cycle", eng2.state["inFlight"] is None) + clock.advance(hours=21) + eng2.tick() + check("kill: the killed image is retried after the sit-out", runner2.scans.count(POSTGRES) == 1) + check("kill: a successful retry clears the failure", targets(eng2)[POSTGRES]["state"] == "scanned") + check("kill: a successful retry clears killedAt", eng2.state["scans"][D4]["killedAt"] is None) + +with tempfile.TemporaryDirectory() as tmp: + eng, runner, clock = engine(tmp) + eng.refresh_inventory() + eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock) + eng2.publish() + check("mutation: a restart with no inFlight fails nothing", targets(eng2)[POSTGRES]["state"] == "pending") + +with tempfile.TemporaryDirectory() as tmp: + # A registry the scanner cannot reach is credential-required, and Trivy is never invoked. + eng, runner, clock = engine(tmp, private=[" GHCR.io ", ""]) + eng.tick() + t = targets(eng) + check("private registry: reported credential-required", t[PRIVATE]["state"] == "credential-required") + check("private registry: the reason names the registry", "ghcr.io" in t[PRIVATE]["reason"]) + check("private registry: Trivy is never invoked for it", PRIVATE not in runner.scans and NIGHTLY not in runner.scans) + check("private registry: other registries still scan", NGINX in runner.scans) + check("private registry: not counted as failed", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0) +with tempfile.TemporaryDirectory() as tmp: + eng, runner, clock = engine(tmp) + eng.tick() + check("mutation: with no private registry configured Trivy is invoked", PRIVATE in runner.scans) + +with tempfile.TemporaryDirectory() as tmp: + # A corrupt state.json degrades the run that rebuilds from it, not every run thereafter. + Path(tmp, "state.json").write_text("{not json") + eng, runner, clock = engine(tmp) + check("loadError: set after a corrupt state.json", eng.state["loadError"]) + eng.tick() + check("loadError: the rebuilding run is degraded", eng.report["run"]["status"] == "degraded") + check("loadError: cleared after a full run with a good inventory", eng.state["loadError"] is None) + clock.advance(hours=21) + eng.tick() + check("loadError: the next full run is complete", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"]) +with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "state.json").write_text("{not json") + eng, runner, clock = engine(tmp) + eng.client.fail = True + eng.tick() + check("mutation: loadError is kept when the inventory failed", eng.state["loadError"]) + +with tempfile.TemporaryDirectory() as tmp: + # lastCompleteRunAt means a report reached the store; a refused report must not advance it. + eng, runner, clock = engine(tmp) + real_publish = report.publish + report.publish = lambda *_: False + try: + eng.tick() + finally: + report.publish = real_publish + check("a refused publish does not advance lastCompleteRunAt", eng.state["lastCompleteRunAt"] is None) + clock.advance(hours=21) + eng.tick() + check("mutation: a successful publish does advance it", eng.state["lastCompleteRunAt"] is not None) + +with tempfile.TemporaryDirectory() as tmp: + # One upstream string costs that string. Each of these used to freeze the whole report. + long_reason = "x" * 3000 + eng, runner, clock = engine(tmp, acc="- id: CVE-2026-0010\n reason: " + long_reason + "\n expires: 2026-12-01\n") + runner.outcomes[NGINX] = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [ + {"VulnerabilityID": "CVE-UP", "Severity": "HIGH", "PkgName": "p", "InstalledVersion": "1", + "PrimaryURL": "HTTPS://upper.example/x", "Title": "t\ud800"}]}]} + eng.tick() + check("hostile strings: the report still publishes", json.loads(Path(tmp, "report.json").read_text())["findings"] and report.validate(eng.report) == []) + up = [f for f in findings(eng, NGINX) if f["id"] == "CVE-UP"][0] + check("hostile strings: an uppercase scheme drops only the URL", up["url"] == "") + check("hostile strings: a lone surrogate is removed", "\ud800" not in up["title"]) + check("hostile strings: an over-long acceptance is ignored, the finding stays open", [f for f in findings(eng, POSTGRES) if f["id"] == "CVE-2026-0010"][0]["status"] != "accepted") + check("hostile strings: an over-long acceptance degrades the run", eng.report["run"]["status"] == "degraded") + check("hostile strings: the degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None) + +with tempfile.TemporaryDirectory() as tmp: + # Isolation: values already in state, whatever put them there, cannot refuse the whole report. + eng, runner, clock = engine(tmp) + eng.tick() + clean = report.build(eng.state, [], [], clock()) + check("mutation: a clean state is not reported as repaired", clean["run"]["status"] == "complete" and clean == report.build(eng.state, [], [], clock())) + record = eng.state["scans"][A1] + record["findings"][0].update(url="HTTP://x", title="t" * 900, severity="weird", cvss=99, package="p" * 900) + eng.state["database"] = {"updatedAt": "not-a-time", "version": "1"} + eng.state["remediation"] = [{"image": "a/b", "state": "pr-open", "url": "HTTP://x", "updatedAt": "2026-11-09T00:00:00Z"}] + target = next(i for i in eng.state["inventory"] if i["id"] == NIGHTLY) + target["id"] = "ghcr.io/" + "o" * 3000 + ":1" + target["image"] = "ghcr.io/" + "o" * 3000 + eng.state["scans"][target["id"]] = eng.state["scans"].pop(NIGHTLY) # a by-tag unit is keyed by its id + built = report.build(eng.state, [], [], clock()) + check("isolation: the repaired report is schema-valid", report.validate(built) == []) + check("isolation: a repair degrades the run", built["run"]["status"] == "degraded") + bad = [f for f in built["findings"] if f["targetId"] == NGINX and f["title"] == "t" * 512][0] + check("isolation: the bad finding survives with sanitised fields", + bad["url"] == "" and bad["severity"] == "UNKNOWN" and bad["cvss"] is None and len(bad["package"]) == 256) + check("isolation: every other finding is untouched", len(built["findings"]) == len(clean["findings"])) + check("isolation: a bad remediation loses only its url", built["remediation"][0]["url"] is None) + check("isolation: an unusable database is reported as unavailable", built["database"]["version"] == "unavailable") + long_ids = [t for t in built["targets"] if len(t["id"]) > 2000] + check("isolation: an over-long target stays listed", len(long_ids) == 1 and len(long_ids[0]["id"]) <= 2048) + check("isolation: findings follow the shortened target id", + any(f["targetId"] == long_ids[0]["id"] for f in built["findings"])) + + +class FakeProcess: + returncode = 0 + + def __init__(self, stdout): + self.stdout = stdout + + +class VersionRunner(trivy.Runner): + def __init__(self, stdout): + super().__init__("/nonexistent", "/nonexistent") + self.stdout = stdout + + def _run(self, args, timeout): + return FakeProcess(self.stdout) + + +def db_info(updated, version): + return VersionRunner(json.dumps({"VulnerabilityDB": {"UpdatedAt": updated, "Version": version}})).db_info() + + +check("db_info rejects an unparseable timestamp", db_info("yesterday", "2") is None) +check("db_info rejects a non-string timestamp", db_info(5, "2") is None) +check("db_info bounds the version", len(db_info("2026-11-09T06:13:52.123456789Z", "v" * 500)["version"]) == 128) +check("mutation: db_info keeps a good database", db_info("2026-11-09T06:13:52Z", "2") == {"updatedAt": "2026-11-09T06:13:52Z", "version": "2"}) +check("a credential failure's reason is bounded", len(trivy.classify_failure("r" * 3000, "401 unauthorized").reason) <= 1024) + +for url, want in (("https://a.test/x", "https://a.test/x"), ("http://a.test/", "http://a.test/"), ("HTTPS://a.test/x", ""), + ("HTTP://a.test/x", ""), ("Https://a.test/x", ""), ("javascript:alert(1)", ""), ("", "")): + check("safe_url " + repr(url), trivy.safe_url(url) == want) +check("every url safe_url keeps satisfies the report schema", + report.re.search(report.SCHEMA["properties"]["findings"]["items"]["properties"]["url"]["pattern"], trivy.safe_url("https://a.test/x"))) + # Recorded Trivy output through the normaliser: untrusted text. parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [ {"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y", diff --git a/platform/vuln-scanner/README.md b/platform/vuln-scanner/README.md index b5f8b1e..f689e28 100644 --- a/platform/vuln-scanner/README.md +++ b/platform/vuln-scanner/README.md @@ -17,17 +17,48 @@ Hand-authored platform tooling (not a catalog service), reconciled by its own no The page is `cve.`, internal only, gated by the per-pod oauth2-proxy sidecar to `platform-admins`. The route is disowned from public DNS and the tunnel 404s it; PiHole answers it on the LAN. -## Operator step: add the Keycloak client (NOT done by the change that added this tree) +## Operator action required: the Keycloak client (R-VULN-04 is NOT delivered until this is done) -This change adds everything around the Keycloak client but not the client. Adding it needs a new key in -`platform/keycloak/keycloak-realm-clients.sops.yaml`, an encrypted file an agent must not decrypt; the -L0 check `keycloak-client-secret-reconciler` then requires the client and that key to appear in the -same change. Until this is done the page cannot complete a login (`invalid_client`, fail-closed); the +Until these steps are done the page cannot complete a login (`invalid_client`, fail-closed). The scanner, its metrics and its alerts are unaffected. -In one PR (the realm import does not re-run on its own for a NEW client): +An agent cannot do this, because every route runs through one encrypted file. The client's secret +must be the key `vuln-scanner-client-secret` in `platform/keycloak/keycloak-realm-clients.sops.yaml`: +the reconciler mounts only that Secret, and the L0 check `keycloak-client-secret-reconciler` rejects a +placeholder that reads any other (a second source would be declared but never healed). That check +also requires the client and the key to land in the same change. Adding a key needs the age private +key. Separately, a completed realm import never re-runs, so a new client has to be created in the +live realm once whatever git says. `*.sops.yaml` is a sensitive path, so the PR needs an approval or +an operator-posted `/approve-override` (`infra/gitea/approval-policy.json`). -1. Generate one value and set it in both encrypted files, through pipes so it is never echoed: +**Order matters.** Create the live client first: the reconciler treats every `*-client-secret` key as +a client that must exist, and fails every 15-minute run with `NOT FOUND` otherwise +(`KeycloakClientSecretReconcilerFailing`). + +1. Create the client in the live realm. No secret is passed: the reconciler sets it from git after + the merge, and it reads the admin credential from the pod's own environment, so nothing is echoed. + + ```bash + kubectl --context k3s-prod -n keycloak exec keycloak-0 -- bash -c ' + K="/opt/keycloak/bin/kcadm.sh"; C="--config /tmp/kcadm-vuln.config" + $K config credentials $C --server http://localhost:8080 --realm master \ + --user "$KC_BOOTSTRAP_ADMIN_USERNAME" --password "$KC_BOOTSTRAP_ADMIN_PASSWORD" + D=home.keithjackson.dev + ID=$($K create clients $C -r homelab -i -s clientId=vuln-scanner -s "name=CVE report (oauth2-proxy)" \ + -s protocol=openid-connect -s enabled=true -s publicClient=false -s standardFlowEnabled=true \ + -s directAccessGrantsEnabled=false -s "redirectUris=[\"https://cve.$D/oauth2/callback\"]" \ + -s "webOrigins=[\"https://cve.$D\"]") + $K create clients/$ID/protocol-mappers/models $C -r homelab -s name=groups -s protocol=openid-connect \ + -s protocolMapper=oidc-group-membership-mapper -s "config.\"full.path\"=false" \ + -s "config.\"id.token.claim\"=true" -s "config.\"access.token.claim\"=true" \ + -s "config.\"userinfo.token.claim\"=true" -s "config.\"claim.name\"=groups" + rm -f /tmp/kcadm-vuln.config' + ``` + + `D` is the prod `BASE_DOMAIN`; check it against the `cluster-vars` ConfigMap first. + +2. On a branch, generate one value and set it in both encrypted files, through pipes so it is never + echoed. `client-secret` in the scanner's file is replaced so that the two match: ```bash ./infra/devcontainer/dev.sh run bash -c ' @@ -38,7 +69,7 @@ In one PR (the realm import does not re-run on its own for a NEW client): "[\"stringData\"][\"client-secret\"]" "\"$cs\""' ``` -2. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client: +3. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client: ```yaml # spec.placeholders @@ -72,9 +103,11 @@ In one PR (the realm import does not re-run on its own for a NEW client): "claim.name": groups ``` -3. Regenerate the derived URL file: `python3 ci/render-keycloak-client-urls.py --write`. Commit all - three, gate, merge. +4. Regenerate the derived URL file with `python3 ci/render-keycloak-client-urls.py --write`, and add + `vuln-scanner-client-secret: REPLACE-ME` to `keycloak-realm-clients.sops.yaml.example`. Commit all of + it in one PR, gate, merge. -4. After merge, the client must also exist in the live realm (a completed realm import does not re-run, - per `platform/keycloak/README.md`, so create it once with `kcadm` or re-trigger the import). The - client-secret reconciler then converges its secret and URLs. +5. After Flux applies it, the next reconciler run sets the client's secret and URLs from git. Prove it + with the L2 test `keycloak-client-secrets-match-git`, then log in at `https://cve.`. The + `platform-admins` restriction is the sidecar's `--allowed-group`; the `groups` mapper above is what + supplies the claim it reads, so a user outside that group gets a 403 from oauth2-proxy. diff --git a/platform/vuln-scanner/deployment.yaml b/platform/vuln-scanner/deployment.yaml index a394837..094908c 100644 --- a/platform/vuln-scanner/deployment.yaml +++ b/platform/vuln-scanner/deployment.yaml @@ -133,6 +133,12 @@ spec: value: /tools/ca-certificates.crt - name: VULN_ACCEPTANCES value: /config/acceptances.yaml + # Registries that need a credential this scanner deliberately does not hold (R-VULN-09). + # Comma-separated hostnames. They report credential-required WITHOUT invoking Trivy: + # the egress policy excludes the LAN, so a real attempt ends in a connection error and + # would read as `failed`, firing VulnCoverageGap permanently. Flux substitutes the domain. + - name: VULN_PRIVATE_REGISTRIES + value: gitea.${BASE_DOMAIN} - name: PYTHONUNBUFFERED value: "1" - name: PYTHONDONTWRITEBYTECODE @@ -222,7 +228,7 @@ spec: env: # client-secret MUST equal the Keycloak `vuln-scanner` client's secret (key # vuln-scanner-client-secret in platform/keycloak/keycloak-realm-clients.sops.yaml; see - # README.md, Operator step). Secrets by name only (R-IDP-04). + # README.md, Operator action required). Secrets by name only (R-IDP-04). - name: OAUTH2_PROXY_CLIENT_ID valueFrom: secretKeyRef: { name: vuln-scanner-oauth2-proxy, key: client-id } diff --git a/platform/vuln-scanner/src/acceptances.py b/platform/vuln-scanner/src/acceptances.py index b8f4cd3..482e49f 100644 --- a/platform/vuln-scanner/src/acceptances.py +++ b/platform/vuln-scanner/src/acceptances.py @@ -15,6 +15,7 @@ from datetime import date, timedelta from state import utcnow MAX_DAYS = 180 +MAX_REASON = 2048 # report.schema.json's limit; a longer reason would make the whole report invalid _KEYS = ("id", "image", "reason", "expires") _ENTRY = re.compile(r"^- ([a-z]+):(?: (.*))?$") _FIELD = re.compile(r"^ ([a-z]+):(?: (.*))?$") @@ -91,6 +92,8 @@ def _validate(fields, today): problems.append("id is missing or malformed") if not fields.get("reason"): problems.append("reason is mandatory") + elif len(fields["reason"]) > MAX_REASON: + problems.append("reason is longer than " + str(MAX_REASON) + " characters") if "image" in fields and not fields["image"]: problems.append("image is empty") expires = fields.get("expires") or "" diff --git a/platform/vuln-scanner/src/main.py b/platform/vuln-scanner/src/main.py index 6e0f3e3..a428d85 100644 --- a/platform/vuln-scanner/src/main.py +++ b/platform/vuln-scanner/src/main.py @@ -23,15 +23,21 @@ log = logging.getLogger("vuln-scanner") FULL_RUN_INTERVAL = timedelta(hours=20) DB_MAX_AGE = timedelta(hours=48) DB_REFRESH_AGE = timedelta(hours=24) +KILLED_REASON = "scanner killed during scan (OOM/eviction?)" class Engine: - def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120): + def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120, + private_registries=()): self.store = st.Store(data_dir) self.data_dir = data_dir self.client = client self.runner = runner self.acceptances_path = acceptances_path + # Registries known to need credentials the scanner deliberately does not hold (R-VULN-09) and, + # being on the LAN, not reachable from its egress policy: a scan would end in a connection + # error (`failed`) rather than a 401 (`credential-required`), so it is never attempted. + self.private_registries = {name.strip().lower() for name in private_registries if name.strip()} self.clock = clock self.tick_seconds = tick_seconds self.publish_seconds = publish_seconds @@ -43,6 +49,18 @@ class Engine: self.loop_thread = None self._last_publish = 0.0 self._last_inventory = None + self._recover_interrupted_scan() + + def _recover_interrupted_scan(self): + """An inFlight key at startup means the last process died inside that scan.""" + key = self.state.get("inFlight") + if not key: + return + log.error("the previous process was killed while scanning %s; marking it failed", key) + record = self.state["scans"].get(key) + if record is not None: + record["killedAt"] = fmt_ts(self.clock()) + self._record_failure(key, "failed", KILLED_REASON) # --- views used by the HTTP servers; none of them waits for a scan -------------------------- def report_bytes(self): @@ -67,16 +85,19 @@ class Engine: # --- persistence ----------------------------------------------------------------------------- def publish(self): + """True when the report was written; False when it was refused and the previous one stands.""" with self.lock: now = self.clock() entries, errors = acceptances.load(self.acceptances_path, now.date()) for problem in errors: log.error("invalid acceptance ignored: %s", problem) built = report_mod.build(self.state, entries, errors, now) - if report_mod.publish(built, self.data_dir): + published = report_mod.publish(built, self.data_dir) + if published: self.report = built self.report_json = report_mod.serialise(built) self._last_publish = time.monotonic() + return published # --- inventory ------------------------------------------------------------------------------- def refresh_inventory(self): @@ -152,6 +173,14 @@ class Engine: return timedelta.max if moment is None else self.clock() - moment def _scan_unit(self, key, reference): + registry = trivy.registry_of(reference) + if registry in self.private_registries or registry.split(":", 1)[0] in self.private_registries: + self._record_failure(key, "credential-required", + "registry " + registry + " is private and the scanner holds no credential for it") + return + with self.lock: + self.state["inFlight"] = key + self.store.save(self.state) try: found = trivy.parse_results(self.runner.scan(reference)) except trivy.ScanFailure as exc: @@ -166,11 +195,24 @@ class Engine: except Exception: # pylint: disable=broad-except log.exception("scan cache cleanup failed") + def _finish_in_flight(self, key): + """Cleared in the same write as the scan's outcome, so no crash can separate the two.""" + if self.state.get("inFlight") == key: + self.state["inFlight"] = None + + def _killed_recently(self, key): + """A unit that killed the scanner sits out full runs for a day: retrying it first on every + restart would OOM again before any later image, which is the wedge this exists to prevent.""" + killed = parse_ts(self.state["scans"][key].get("killedAt")) + return killed is not None and self.clock() - killed < FULL_RUN_INTERVAL + def _record_failure(self, key, kind, reason): now = fmt_ts(self.clock()) with self.lock: + self._finish_in_flight(key) record = self.state["scans"].get(key) if record is None: + self.store.save(self.state) return record["lastAttemptAt"] = now record["outcome"] = kind @@ -181,10 +223,13 @@ class Engine: def _record_success(self, key, found): now = fmt_ts(self.clock()) with self.lock: + self._finish_in_flight(key) record = self.state["scans"].get(key) if record is None: + self.store.save(self.state) return record["lastAttemptAt"] = now + record["killedAt"] = None record["lastSuccessAt"] = now record["outcome"] = "ok" record["reason"] = None @@ -215,7 +260,10 @@ class Engine: started = self.clock() log.info("full run starting") database_ok = self._refresh_database(always=True) - self._scan_queue(sorted(self._units())) + self._scan_queue([key for key in sorted(self._units()) if not self._killed_recently(key)]) + if self.stop.is_set(): + log.info("full run interrupted by shutdown; not recorded") + return info = self.state["database"] now = self.clock() degraded = not database_ok or info is None or self._db_age(info) > DB_MAX_AGE @@ -225,8 +273,15 @@ class Engine: status = "degraded" if degraded else "complete" with self.lock: self.state["run"] = {"startedAt": fmt_ts(started), "finishedAt": fmt_ts(now), "status": status} - if not degraded: + self.store.save(self.state) + # Only a report that actually reached the store counts as a completed run. Otherwise a + # refused report would freeze the page while VulnScanStale stayed quiet. + if self.publish() and not degraded and self.report["run"]["status"] == "complete": self.state["lastCompleteRunAt"] = fmt_ts(now) + # A corrupt state.json degrades the run that rebuilds from it, and no further: once a full + # run has completed against a good inventory, the state is whole again. + if self.state.get("loadError") and self._last_inventory is not None and not self.state.get("inventoryError"): + self.state["loadError"] = None self.store.save(self.state) log.info("full run finished: %s", status) @@ -262,7 +317,9 @@ def main(): tools_dir = os.environ.get("VULN_TOOLS_DIR", "/tools") trivy_bin = os.environ.get("TRIVY_BIN", os.path.join(tools_dir, "trivy")) runner = trivy.Runner(trivy_bin, cache_dir, int(os.environ.get("VULN_SCAN_TIMEOUT_SECONDS", "600"))) - engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml")) + private_registries = os.environ.get("VULN_PRIVATE_REGISTRIES", "").split(",") + engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"), + private_registries=private_registries) server.start_all(engine, ("127.0.0.1", 8080), ("0.0.0.0", 8081), ("0.0.0.0", 9090)) finished = threading.Event() signal.signal(signal.SIGTERM, lambda *_: finished.set()) diff --git a/platform/vuln-scanner/src/page.py b/platform/vuln-scanner/src/page.py index a2cd1a3..a8bd7b7 100644 --- a/platform/vuln-scanner/src/page.py +++ b/platform/vuln-scanner/src/page.py @@ -180,14 +180,39 @@ def _sev(severity): return '' + e(severity) + "" -def _workload_lines(target): +def _state_label(target): + return "target " + e(target["state"]) + (" (" + e(target["reason"]) + ")" if target.get("reason") else "") + + +def _status_label(finding): + """One finding's own status, with its own acceptance: never another image's.""" + label = "status " + e(finding["status"]) + acceptance = finding.get("acceptance") + if acceptance: + label += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")" + return label + + +def _workload_lines(target, finding=None): + """One line per workload, each carrying the target state and, in a CVE expansion, the status.""" + suffix = " — " + _state_label(target) + (" — " + _status_label(finding) if finding else "") items = [] for workload in target["workloads"]: items.append("
  • " + e(workload["namespace"]) + " / " + e(workload["kind"]) + " / " + e(workload["name"]) - + " — containers: " + e(", ".join(workload["containers"]) or "none") + "
    • ") + + " — containers: " + e(", ".join(workload["containers"]) or "none") + suffix + "") return "
      " + "".join(items) + "
    " +def _status_summary(statuses): + """The row-level status. A CVE accepted on one image and open on another reads as mixed.""" + counts = {} + for status in statuses: + counts[status] = counts.get(status, 0) + 1 + if len(counts) == 1: + return e(next(iter(counts))) + return "mixed: " + ", ".join(e(counts[s]) + " " + e(s) for s in sorted(counts, key=STATUS_RANK.get)) + + def _remediation_cell(images, remediation): if not images: return "" @@ -250,13 +275,10 @@ def _cves(report, targets, params, remediation): for finding, target in row["items"]: if finding["status"] == "fix-available": fix_images.add(target["image"]) - lines.append("
  • " + e(target["id"]) + " [" + e(target["state"]) + "] — " + e(finding["package"]) + " " - + e(finding["installedVersion"]) + " → " + e(finding["fixedVersion"] or "no fix") - + _workload_lines(target) + "
    • ") - acceptance = next((f.get("acceptance") for f, _ in row["items"] if f.get("acceptance")), None) - status_text = e(row["status"]) - if acceptance: - status_text += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")" + lines.append("
  • " + e(target["id"]) + " — " + _state_label(target) + " — " + _status_label(finding) + + " — " + e(finding["package"]) + " " + e(finding["installedVersion"]) + " → " + + e(finding["fixedVersion"] or "no fix") + _workload_lines(target, finding) + "
    • ") + status_text = _status_summary(f["status"] for f, _ in row["items"]) html_rows.append( "" + _link(row["url"], row["id"]) + "" + _sev(row["severity"]) + "" + e("" if row["cvss"] is None else row["cvss"]) + "" + e(row["package"]) + "" + e(row["fix"]) @@ -317,12 +339,13 @@ def _workloads(report, targets, params): html_rows = [] for row in page_rows: lines = "".join( - "
  • " + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + e(f["status"]) + "
    • " - for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"]))) + "
  • " + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + _status_label(f) + + " — " + e(f["targetId"]) + " — " + _state_label(targets[f["targetId"]]) + "
    • " + for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"], f["targetId"]))) counts = "".join("" + e(row["counts"][s]) + "" for s in SEVERITIES) html_rows.append( "" + e(row["key"][0]) + "" + e(row["key"][1]) + "" + e(row["key"][2]) + "" - + e(", ".join(sorted(row["containers"])) or "none") + "" + "
    ".join(e(i) for i in sorted(row["images"])) + "" + + e(", ".join(sorted(row["containers"])) or "none") + "" + "
    ".join(e(i) + " — " + _state_label(targets[i]) for i in sorted(row["images"])) + "" + counts + "
    " + e(len(row["findings"])) + " findings
      " + lines + "
    ") head = [_th(params, "Namespace", "namespace"), _th(params, "Kind", "kind"), _th(params, "Workload", "name"), "Containers", _th(params, "Images", "images")] + [_th(params, s, s.lower()) for s in SEVERITIES] + [_th(params, "Findings", "total")] diff --git a/platform/vuln-scanner/src/report.py b/platform/vuln-scanner/src/report.py index bb5e213..9dad978 100644 --- a/platform/vuln-scanner/src/report.py +++ b/platform/vuln-scanner/src/report.py @@ -1,6 +1,7 @@ """Build, validate and publish report.json, the one interface to the page and the bot.""" import json import logging +import hashlib import os import re from datetime import date, timedelta @@ -85,10 +86,15 @@ def build(state, accepted, acceptance_errors, now): if acceptance_errors or state.get("inventoryError") or state.get("loadError") \ or db_time is None or now - db_time > DB_STALE: run["status"] = "degraded" - return { + built = { "schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run, "targets": targets, "findings": findings, "remediation": list(state["remediation"]), } + repaired = _isolate(built) + if repaired: + log.error("%d report entries held a value the schema refuses; repaired or dropped them", repaired) + run["status"] = "degraded" + return built # --- hand validator for the JSON-Schema subset report.schema.json uses ----------------------------- @@ -175,6 +181,88 @@ def _check(schema, value, path, errors): _check(branch, value, path, errors) +# --- isolation: one bad upstream value costs that value, never the whole report --------------------- + +def _repair(schema, value): + """(usable, value): value itself when valid, else the nearest valid form of it.""" + trial = [] + _check(schema, value, "$", trial) + if not trial: + return True, value + if isinstance(value, dict) and "properties" in schema: + fixed = {} + for key, sub in schema["properties"].items(): + if key not in value: + continue + usable, item = _repair(sub, value[key]) + if usable: + fixed[key] = item + elif key in schema.get("required", []): + return False, None + trial = [] + _check(schema, fixed, "$", trial) + return not trial, fixed + if isinstance(value, list) and "items" in schema: + repaired = [_repair(schema["items"], item) for item in value] + return True, [item for usable, item in repaired if usable] + candidates = [None, "", "UNKNOWN"] + if isinstance(value, str) and "maxLength" in schema: + candidates.insert(0, value[:schema["maxLength"]]) + for candidate in candidates: + trial = [] + _check(schema, candidate, "$", trial) + if not trial: + return True, candidate + return False, None + + +def _isolate(report): + """Repair (or, as a last resort, drop) each entry the schema refuses. Returns how many changed.""" + changed = 0 + schema = SCHEMA["properties"] + usable, database = _repair(schema["database"], report["database"]) + if database is not report["database"]: + changed += 1 + report["database"] = database if usable else {"updatedAt": EPOCH, "version": "unavailable"} + renamed, dropped, targets = {}, set(), [] + for target in report["targets"]: + original = target["id"] + if len(original) > 2048: # keep it unique: a truncated id may not collide with another + target = dict(target, id=original[:2031] + "#" + hashlib.sha256(original.encode("utf-8")).hexdigest()[:16]) + renamed[original] = target["id"] + usable, fixed = _repair(schema["targets"]["items"], target) + if not usable: + changed += 1 + dropped.add(original) + continue + if fixed is not target or original in renamed: + changed += 1 + targets.append(fixed) + report["targets"] = targets + findings = [] + for finding in report["findings"]: + if finding["targetId"] in dropped: + changed += 1 + continue + if finding["targetId"] in renamed: + finding = dict(finding, targetId=renamed[finding["targetId"]]) + usable, fixed = _repair(schema["findings"]["items"], finding) + if not usable or fixed is not finding: + changed += 1 + if usable: + findings.append(fixed) + report["findings"] = findings + remediation = [] + for entry in report["remediation"]: + usable, fixed = _repair(schema["remediation"]["items"], entry) + if not usable or fixed is not entry: + changed += 1 + if usable: + remediation.append(fixed) + report["remediation"] = remediation + return changed + + def validate(report): """Schema and referential errors; an empty list means the report may be served.""" errors = [] diff --git a/platform/vuln-scanner/src/server.py b/platform/vuln-scanner/src/server.py index 7746bcf..09aadff 100644 --- a/platform/vuln-scanner/src/server.py +++ b/platform/vuln-scanner/src/server.py @@ -126,7 +126,9 @@ def validate_triage(document): if entry["state"] not in REMEDIATION_STATES: return [prefix + "unknown state"] url = entry["url"] - if url is not None and (not isinstance(url, str) or len(url) > 2048 or trivy.safe_url(url) != url): + # safe_url is the same rule the report schema enforces, so a value accepted here cannot make + # the report invalid. Empty is refused: safe_url returns "" for it, and the schema does not. + if url is not None and (not isinstance(url, str) or not url or trivy.safe_url(url) != url): return [prefix + "url must be null or an http(s) URL"] if parse_ts(entry["updatedAt"]) is None or len(entry["updatedAt"]) > 64: return [prefix + "updatedAt must be RFC 3339"] diff --git a/platform/vuln-scanner/src/state.py b/platform/vuln-scanner/src/state.py index 56912db..35dcad8 100644 --- a/platform/vuln-scanner/src/state.py +++ b/platform/vuln-scanner/src/state.py @@ -72,6 +72,9 @@ def new_state(): "remediation": [], "triageHeartbeatAt": None, "loadError": None, + # The scan key Trivy is running against right now, persisted BEFORE the scan starts: an OOM + # kill takes the whole container, so nothing can record the failure afterwards. + "inFlight": None, } @@ -84,6 +87,7 @@ def new_record(now): "outcome": None, "reason": None, "failedSince": None, + "killedAt": None, "findings": [], "firstSeen": {}, } diff --git a/platform/vuln-scanner/src/trivy.py b/platform/vuln-scanner/src/trivy.py index 2481965..21a79fb 100644 --- a/platform/vuln-scanner/src/trivy.py +++ b/platform/vuln-scanner/src/trivy.py @@ -5,6 +5,8 @@ import re import subprocess from urllib.parse import urlsplit +from state import parse_ts + log = logging.getLogger("vuln-scanner.trivy") SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN") @@ -13,7 +15,11 @@ _CREDENTIAL = re.compile( r"|status(?: code)?:? ?(?:401|403)\b|\b40[13] (?:unauthorized|forbidden)", re.IGNORECASE, ) -_CONTROL = re.compile(r"[\x00-\x1f\x7f]") +# Lone surrogates too: json.loads accepts them, and they cannot be encoded as UTF-8 for the page. +_CONTROL = re.compile(r"[\x00-\x1f\x7f\ud800-\udfff]") +# Exact and lowercase, matching report.schema.json, whose pattern is case-sensitive: an `HTTPS://x` +# accepted here would make the schema refuse the entire report. +_URL_SCHEME = re.compile(r"https?://") class ScanFailure(Exception): @@ -35,13 +41,18 @@ def text(value, limit=512): def safe_url(value): """value if it is a plain http(s) URL, else the empty string.""" url = text(value, 2048) - if not url or url != value or re.search(r"\s", url): + if not url or url != value or re.search(r"\s", url) or not _URL_SCHEME.match(url): return "" try: parts = urlsplit(url) except ValueError: return "" - return url if parts.scheme.lower() in ("http", "https") and parts.netloc else "" + return url if parts.netloc else "" + + +def registry_of(reference): + """The lowercase registry host (and port) of a normalised registry/repository reference.""" + return reference.split("/", 1)[0].lower() def _cvss(vuln): @@ -95,7 +106,7 @@ def parse_results(document): def classify_failure(registry, stderr): """Sort a non-zero Trivy exit into credential-required or failed. Reasons name the registry.""" if _CREDENTIAL.search(stderr or ""): - return ScanFailure("credential-required", "registry " + registry + " refused anonymous access") + return ScanFailure("credential-required", "registry " + text(registry, 253) + " refused anonymous access") lines = [line.strip() for line in (stderr or "").splitlines() if line.strip()] return ScanFailure("failed", text(lines[-1], 400) if lines else "trivy exited with an error and no message") @@ -125,9 +136,9 @@ class Runner: db = json.loads(result.stdout).get("VulnerabilityDB") except (ValueError, AttributeError): return None - if not isinstance(db, dict) or not db.get("UpdatedAt"): - return None - return {"updatedAt": db["UpdatedAt"], "version": str(db.get("Version", "unknown"))} + if not isinstance(db, dict) or parse_ts(db.get("UpdatedAt")) is None: + return None # an unparseable timestamp is no usable database, and must not reach the report + return {"updatedAt": db["UpdatedAt"], "version": text(str(db.get("Version", "unknown")), 128) or "unknown"} def scan(self, reference): args = [ @@ -143,7 +154,7 @@ class Runner: if result.returncode < 0: raise ScanFailure("failed", "trivy killed by signal " + str(-result.returncode) + " (out of memory?)") if result.returncode != 0: - raise classify_failure(reference.split("/", 1)[0], result.stderr) + raise classify_failure(registry_of(reference), result.stderr) try: return json.loads(result.stdout) except ValueError: diff --git a/renovate.json5 b/renovate.json5 index 98554a3..9bd46ba 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -483,6 +483,16 @@ "description": "ADDED 2026-10-30. MAJORS WAIT FOR A TICK ON THE DEPENDENCY DASHBOARD instead of opening a PR. They are never auto-merged anyway, and with prConcurrentLimit 5 the dry run's 10 majors would hold every PR slot open indefinitely, starving the patch/minor/digest PRs that can merge on green.", "matchUpdateTypes": ["major"], "dependencyDashboardApproval": true + }, + // ADDED 2026-11-09 — EVERY IMAGE IN THE SCANNER POD, NOT ONLY TRIVY. The python image runs the + // process that holds the cluster-read token (rbac.yaml, R-VULN-09), and without this rule its + // digest and patch bumps fall through to the auto-merge rule above. LAST ON PURPOSE: Renovate + // applies packageRules in order and a later match overrides, so nothing above can undo it. + { + "description": "Anything under platform/vuln-scanner: the python image holds the cluster-read token. Never auto-merged, 7-day minimum release age (R-VULN-09).", + "matchFileNames": ["platform/vuln-scanner/**"], + "automerge": false, + "minimumReleaseAge": "7 days" } ], // WHAT RENOVATE STILL CANNOT FIX, so nobody mistakes a working bot for full coverage. A