Merge branch 'wip/vuln-fixes' into feat/vuln-visibility
This commit is contained in:
commit
4ca0ebfac6
15 files changed
+557
-43
No files matched your search
@@ -16,6 +16,11 @@
|
||||
# (6) the pod: non-root, read-only rootfs, drop ALL, seccomp RuntimeDefault, no host
|
||||
# namespaces or hostPath, CPU and memory limits on every container, an ephemeral-storage
|
||||
# limit on the scanner, and a size-limited /cache emptyDir.
|
||||
# (8) a renovate.json5 rule covers every file under platform/vuln-scanner: automerge:false,
|
||||
# minimumReleaseAge of at least 7 days, and no later rule turns auto-merge back on (the
|
||||
# python image holds the cluster-read token);
|
||||
# (9) the scanner names the private registry it cannot reach, so that registry reports
|
||||
# credential-required rather than a permanent failed.
|
||||
# AND (7) each evaluator is run on a mutated copy and must flag it (non-vacuity).
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
@@ -150,6 +155,33 @@ def eval_renovate(rules):
|
||||
return bad
|
||||
|
||||
|
||||
SCANNER_FILES = "platform/vuln-scanner/**"
|
||||
|
||||
|
||||
def eval_renovate_files(rules):
|
||||
positions = [i for i, r in enumerate(rules) if SCANNER_FILES in (r.get("matchFileNames") or [])]
|
||||
if not positions:
|
||||
return [f"no packageRule has matchFileNames {SCANNER_FILES}"]
|
||||
bad = []
|
||||
rule = rules[positions[-1]]
|
||||
if rule.get("automerge") is not False:
|
||||
bad.append("the platform/vuln-scanner file rule is not automerge:false")
|
||||
age = VERSION_UPDATE_RULE_RE.match(rule.get("minimumReleaseAge") or "")
|
||||
if not age or int(age.group(1)) < 7:
|
||||
bad.append("the platform/vuln-scanner file rule has no minimumReleaseAge of at least 7 days")
|
||||
if any(r.get("automerge") is True for r in rules[positions[-1] + 1:]):
|
||||
bad.append("a rule after the platform/vuln-scanner file rule re-enables auto-merge")
|
||||
return bad
|
||||
|
||||
|
||||
def eval_private_registries(pod):
|
||||
scanner = next(c for c in pod["containers"] if c["name"] == "scanner")
|
||||
env = {e["name"]: e.get("value") for e in scanner.get("env") or []}
|
||||
if env.get("VULN_PRIVATE_REGISTRIES") != "gitea.${BASE_DOMAIN}":
|
||||
return ["the scanner does not name the private registry gitea.${BASE_DOMAIN}"]
|
||||
return []
|
||||
|
||||
|
||||
def eval_rbac(rbac):
|
||||
bad = []
|
||||
croles = [d for d in rbac if d.get("kind") == "ClusterRole"]
|
||||
@@ -224,6 +256,10 @@ rules = renovate.get("packageRules") or []
|
||||
check(len(rules) > 5, "(4) parsed renovate.json5 packageRules (parser degenerated)")
|
||||
for v in eval_renovate(rules):
|
||||
check(False, f"(4) {v}")
|
||||
for v in eval_renovate_files(rules):
|
||||
check(False, f"(8) {v}")
|
||||
for v in eval_private_registries(pod):
|
||||
check(False, f"(9) {v}")
|
||||
|
||||
rbac = docs(COMP / "rbac.yaml")
|
||||
for v in eval_rbac(rbac):
|
||||
@@ -274,6 +310,23 @@ for name, fn in (("Trivy automerge flipped true", ren_auto), ("minimumReleaseAge
|
||||
("Trivy removed from Renovate rules", ren_drop), ("later auto-merge rule appended", ren_later)):
|
||||
check(bool(eval_renovate(mutated(fn, rules))), f"(7) Renovate evaluator missed mutant: {name}")
|
||||
|
||||
# Renovate file-rule mutants
|
||||
def files_rule(r):
|
||||
return next(x for x in r if SCANNER_FILES in (x.get("matchFileNames") or []))
|
||||
def files_auto(r): files_rule(r)["automerge"] = True
|
||||
def files_age(r): files_rule(r)["minimumReleaseAge"] = "1 day"
|
||||
def files_drop(r): r.remove(files_rule(r))
|
||||
def files_later(r): r.append({"matchDatasources": ["docker"], "automerge": True})
|
||||
for name, fn in (("file rule automerge true", files_auto), ("file rule age shortened", files_age),
|
||||
("file rule removed", files_drop), ("later rule re-enables auto-merge", files_later)):
|
||||
check(bool(eval_renovate_files(mutated(fn, rules))), f"(7) file-rule evaluator missed mutant: {name}")
|
||||
|
||||
def reg_drop(p):
|
||||
next(c for c in p["containers"] if c["name"] == "scanner")["env"][:] = [
|
||||
e for e in next(c for c in p["containers"] if c["name"] == "scanner")["env"]
|
||||
if e["name"] != "VULN_PRIVATE_REGISTRIES"]
|
||||
check(bool(eval_private_registries(mutated(reg_drop, pod))), "(7) private-registry evaluator missed the removal")
|
||||
|
||||
# Pod mutants
|
||||
def pod_root(p): p["securityContext"]["runAsNonRoot"] = False
|
||||
def pod_host(p): p["hostNetwork"] = True
|
||||
|
||||
@@ -94,6 +94,15 @@ def status(when, accepted, errs=()):
|
||||
return built, built["findings"][0]
|
||||
|
||||
|
||||
def reason_entry(length):
|
||||
return acceptances.parse_text("- id: CVE-1\n reason: " + "r" * length + "\n expires: 2026-12-01\n", NOW.date())
|
||||
|
||||
|
||||
found, problems = reason_entry(3000)
|
||||
check("a reason over 2048 characters ignores the entry and reports it", found == [] and any("2048" in p for p in problems))
|
||||
found, problems = reason_entry(2048)
|
||||
check("mutation: a reason of exactly 2048 characters is valid", len(found) == 1 and problems == [])
|
||||
|
||||
entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)}
|
||||
built, finding = status(NOW, [entry])
|
||||
check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted"
|
||||
|
||||
@@ -150,6 +150,10 @@ with tempfile.TemporaryDirectory() as tmp:
|
||||
mutate(document)
|
||||
status, _, _ = post(internal, document)
|
||||
check("triage rejects " + name, status == 400)
|
||||
for name, url in (("an uppercase scheme", "HTTP://gitea.test/pulls/1"), ("an empty url", ""), ("a javascript url", "javascript:alert(1)")):
|
||||
bad = json.loads(json.dumps(valid))
|
||||
bad["remediation"][0]["url"] = url
|
||||
check("triage rejects " + name, post(internal, bad)[0] == 400)
|
||||
check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415)
|
||||
status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"})
|
||||
check("triage rejects an oversized body", status == 413)
|
||||
|
||||
@@ -170,6 +170,39 @@ _, _, body = render()
|
||||
check("default view excludes nothing", all(f["id"] in body for f in REPORT["findings"]))
|
||||
check("remediation column shows pr-open with a link", 'href="https://gitea.test/pulls/1"' in body and "pr-open" in body)
|
||||
|
||||
# Status and state belong to the line they describe (R-VULN-04).
|
||||
images = [t for t in REPORT["targets"] if t["type"] == "image" and t["workloads"]]
|
||||
check("fixture has two image targets with workloads", len(images) >= 2)
|
||||
mixed = copy.deepcopy(REPORT)
|
||||
first, second = next(t for t in mixed["targets"] if t["id"] == images[0]["id"]), next(t for t in mixed["targets"] if t["id"] == images[1]["id"])
|
||||
first["state"], first["reason"] = "scanned", None
|
||||
second["state"], second["reason"] = "stale", "last success long ago"
|
||||
base = dict(REPORT["findings"][0], id="CVE-MIXED")
|
||||
base.pop("acceptance", None)
|
||||
mixed["findings"] = [
|
||||
dict(base, targetId=first["id"], status="accepted", acceptance={"reason": "only-on-the-first", "expires": "2026-12-01"}),
|
||||
dict(base, targetId=second["id"], status="fix-available"),
|
||||
]
|
||||
_, _, body = render(mixed, "view=cves&q=CVE-MIXED")
|
||||
check("a mixed CVE reads as mixed, with counts", "mixed: 1 fix-available, 1 accepted" in body)
|
||||
check("a mixed CVE's row does not carry one image's acceptance", "<td>mixed: 1 fix-available, 1 accepted</td>" in body)
|
||||
first_line = page.e(first["id"]) + " — target scanned — status accepted (only-on-the-first, until 2026-12-01)"
|
||||
second_line = page.e(second["id"]) + " — target stale (last success long ago) — status fix-available"
|
||||
check("the accepted image's line carries its own acceptance", first_line in body)
|
||||
check("the other image's line carries its own status and state", second_line in body)
|
||||
check("no line attaches the acceptance to the unaccepted status", "status fix-available (only-on-the-first" not in body)
|
||||
check("workload lines inside the expansion carry state and status",
|
||||
"target stale (last success long ago) — status fix-available</li>" in body
|
||||
and "containers: " + page.e(", ".join(second["workloads"][0]["containers"]) or "none") + " — target stale" in body)
|
||||
_, _, plain = render(query="view=cves&status=accepted")
|
||||
check("mutation: a uniformly accepted CVE does not read as mixed", "mixed:" not in plain and "CVE-2026-1003" in plain)
|
||||
_, _, body = render(mixed, "view=workloads")
|
||||
check("the workloads images column labels each image's state", page.e(second["id"]) + " — target stale (last success long ago)" in body)
|
||||
check("the workloads expansion labels each finding's status, image and state",
|
||||
"status fix-available — " + page.e(second["id"]) + " — target stale (last success long ago)</li>" in body)
|
||||
check("the workloads expansion keeps an acceptance on its own line",
|
||||
"status accepted (only-on-the-first, until 2026-12-01) — " + page.e(first["id"]) + " — target scanned</li>" in body)
|
||||
|
||||
big = copy.deepcopy(REPORT)
|
||||
big["findings"] = [dict(REPORT["findings"][0], id="CVE-2026-" + format(i, "05d")) for i in range(250)]
|
||||
_, _, body = render(big, "view=cves&size=50&page=2")
|
||||
|
||||
@@ -95,14 +95,14 @@ class FakeRunner:
|
||||
pass
|
||||
|
||||
|
||||
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER):
|
||||
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER, private=()):
|
||||
path = os.path.join(tmp, "acceptances.yaml")
|
||||
if not os.path.exists(path):
|
||||
Path(path).write_text(acc)
|
||||
clock = clock or Clock()
|
||||
runner = runner or FakeRunner()
|
||||
runner.clock = clock
|
||||
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock
|
||||
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0, private_registries=private), runner, clock
|
||||
|
||||
|
||||
def targets(eng):
|
||||
@@ -239,6 +239,184 @@ with tempfile.TemporaryDirectory() as tmp:
|
||||
thread.join(10)
|
||||
check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive())
|
||||
|
||||
class Killed(BaseException):
|
||||
"""What an OOM kill looks like from inside the process: no `except Exception` runs."""
|
||||
|
||||
|
||||
class KillingRunner(FakeRunner):
|
||||
def __init__(self, victim, data_dir):
|
||||
super().__init__()
|
||||
self.victim = victim
|
||||
self.data_dir = data_dir
|
||||
self.on_disk = None
|
||||
|
||||
def scan(self, reference):
|
||||
if reference == self.victim:
|
||||
self.on_disk = json.loads(Path(self.data_dir, "state.json").read_text()).get("inFlight")
|
||||
raise Killed()
|
||||
return super().scan(reference)
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# R-VULN-05 BLOCKER: a scan that kills the container must cost that image only.
|
||||
eng, runner, clock = engine(tmp, runner=KillingRunner(POSTGRES, tmp))
|
||||
killed = False
|
||||
try:
|
||||
eng.tick()
|
||||
except Killed:
|
||||
killed = True
|
||||
check("kill: the simulated OOM really aborted the process path", killed)
|
||||
check("kill: inFlight is on disk BEFORE the scan starts", runner.on_disk == D4)
|
||||
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
|
||||
check("kill: startup clears inFlight", eng2.state["inFlight"] is None)
|
||||
check("kill: clearing is persisted", json.loads(Path(tmp, "state.json").read_text())["inFlight"] is None)
|
||||
eng2.publish()
|
||||
t = targets(eng2)
|
||||
check("kill: the target is failed", t[POSTGRES]["state"] == "failed")
|
||||
check("kill: the reason is the killed-scan text", t[POSTGRES]["reason"] == main.KILLED_REASON)
|
||||
eng2.tick()
|
||||
t = targets(eng2)
|
||||
check("kill: images after the killed one get scanned", PRIVATE in runner2.scans and t[PRIVATE]["state"] == "credential-required")
|
||||
check("kill: the killed image is not retried on restart (no wedge)", POSTGRES not in runner2.scans)
|
||||
check("kill: the killed target stays failed with its reason", t[POSTGRES]["reason"] == main.KILLED_REASON)
|
||||
check("kill: inFlight is clear after a normal cycle", eng2.state["inFlight"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng2.tick()
|
||||
check("kill: the killed image is retried after the sit-out", runner2.scans.count(POSTGRES) == 1)
|
||||
check("kill: a successful retry clears the failure", targets(eng2)[POSTGRES]["state"] == "scanned")
|
||||
check("kill: a successful retry clears killedAt", eng2.state["scans"][D4]["killedAt"] is None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.refresh_inventory()
|
||||
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
|
||||
eng2.publish()
|
||||
check("mutation: a restart with no inFlight fails nothing", targets(eng2)[POSTGRES]["state"] == "pending")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A registry the scanner cannot reach is credential-required, and Trivy is never invoked.
|
||||
eng, runner, clock = engine(tmp, private=[" GHCR.io ", ""])
|
||||
eng.tick()
|
||||
t = targets(eng)
|
||||
check("private registry: reported credential-required", t[PRIVATE]["state"] == "credential-required")
|
||||
check("private registry: the reason names the registry", "ghcr.io" in t[PRIVATE]["reason"])
|
||||
check("private registry: Trivy is never invoked for it", PRIVATE not in runner.scans and NIGHTLY not in runner.scans)
|
||||
check("private registry: other registries still scan", NGINX in runner.scans)
|
||||
check("private registry: not counted as failed", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
check("mutation: with no private registry configured Trivy is invoked", PRIVATE in runner.scans)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# A corrupt state.json degrades the run that rebuilds from it, not every run thereafter.
|
||||
Path(tmp, "state.json").write_text("{not json")
|
||||
eng, runner, clock = engine(tmp)
|
||||
check("loadError: set after a corrupt state.json", eng.state["loadError"])
|
||||
eng.tick()
|
||||
check("loadError: the rebuilding run is degraded", eng.report["run"]["status"] == "degraded")
|
||||
check("loadError: cleared after a full run with a good inventory", eng.state["loadError"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng.tick()
|
||||
check("loadError: the next full run is complete", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
Path(tmp, "state.json").write_text("{not json")
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.client.fail = True
|
||||
eng.tick()
|
||||
check("mutation: loadError is kept when the inventory failed", eng.state["loadError"])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# lastCompleteRunAt means a report reached the store; a refused report must not advance it.
|
||||
eng, runner, clock = engine(tmp)
|
||||
real_publish = report.publish
|
||||
report.publish = lambda *_: False
|
||||
try:
|
||||
eng.tick()
|
||||
finally:
|
||||
report.publish = real_publish
|
||||
check("a refused publish does not advance lastCompleteRunAt", eng.state["lastCompleteRunAt"] is None)
|
||||
clock.advance(hours=21)
|
||||
eng.tick()
|
||||
check("mutation: a successful publish does advance it", eng.state["lastCompleteRunAt"] is not None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# One upstream string costs that string. Each of these used to freeze the whole report.
|
||||
long_reason = "x" * 3000
|
||||
eng, runner, clock = engine(tmp, acc="- id: CVE-2026-0010\n reason: " + long_reason + "\n expires: 2026-12-01\n")
|
||||
runner.outcomes[NGINX] = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-UP", "Severity": "HIGH", "PkgName": "p", "InstalledVersion": "1",
|
||||
"PrimaryURL": "HTTPS://upper.example/x", "Title": "t\ud800"}]}]}
|
||||
eng.tick()
|
||||
check("hostile strings: the report still publishes", json.loads(Path(tmp, "report.json").read_text())["findings"] and report.validate(eng.report) == [])
|
||||
up = [f for f in findings(eng, NGINX) if f["id"] == "CVE-UP"][0]
|
||||
check("hostile strings: an uppercase scheme drops only the URL", up["url"] == "")
|
||||
check("hostile strings: a lone surrogate is removed", "\ud800" not in up["title"])
|
||||
check("hostile strings: an over-long acceptance is ignored, the finding stays open", [f for f in findings(eng, POSTGRES) if f["id"] == "CVE-2026-0010"][0]["status"] != "accepted")
|
||||
check("hostile strings: an over-long acceptance degrades the run", eng.report["run"]["status"] == "degraded")
|
||||
check("hostile strings: the degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
# Isolation: values already in state, whatever put them there, cannot refuse the whole report.
|
||||
eng, runner, clock = engine(tmp)
|
||||
eng.tick()
|
||||
clean = report.build(eng.state, [], [], clock())
|
||||
check("mutation: a clean state is not reported as repaired", clean["run"]["status"] == "complete" and clean == report.build(eng.state, [], [], clock()))
|
||||
record = eng.state["scans"][A1]
|
||||
record["findings"][0].update(url="HTTP://x", title="t" * 900, severity="weird", cvss=99, package="p" * 900)
|
||||
eng.state["database"] = {"updatedAt": "not-a-time", "version": "1"}
|
||||
eng.state["remediation"] = [{"image": "a/b", "state": "pr-open", "url": "HTTP://x", "updatedAt": "2026-11-09T00:00:00Z"}]
|
||||
target = next(i for i in eng.state["inventory"] if i["id"] == NIGHTLY)
|
||||
target["id"] = "ghcr.io/" + "o" * 3000 + ":1"
|
||||
target["image"] = "ghcr.io/" + "o" * 3000
|
||||
eng.state["scans"][target["id"]] = eng.state["scans"].pop(NIGHTLY) # a by-tag unit is keyed by its id
|
||||
built = report.build(eng.state, [], [], clock())
|
||||
check("isolation: the repaired report is schema-valid", report.validate(built) == [])
|
||||
check("isolation: a repair degrades the run", built["run"]["status"] == "degraded")
|
||||
bad = [f for f in built["findings"] if f["targetId"] == NGINX and f["title"] == "t" * 512][0]
|
||||
check("isolation: the bad finding survives with sanitised fields",
|
||||
bad["url"] == "" and bad["severity"] == "UNKNOWN" and bad["cvss"] is None and len(bad["package"]) == 256)
|
||||
check("isolation: every other finding is untouched", len(built["findings"]) == len(clean["findings"]))
|
||||
check("isolation: a bad remediation loses only its url", built["remediation"][0]["url"] is None)
|
||||
check("isolation: an unusable database is reported as unavailable", built["database"]["version"] == "unavailable")
|
||||
long_ids = [t for t in built["targets"] if len(t["id"]) > 2000]
|
||||
check("isolation: an over-long target stays listed", len(long_ids) == 1 and len(long_ids[0]["id"]) <= 2048)
|
||||
check("isolation: findings follow the shortened target id",
|
||||
any(f["targetId"] == long_ids[0]["id"] for f in built["findings"]))
|
||||
|
||||
|
||||
class FakeProcess:
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, stdout):
|
||||
self.stdout = stdout
|
||||
|
||||
|
||||
class VersionRunner(trivy.Runner):
|
||||
def __init__(self, stdout):
|
||||
super().__init__("/nonexistent", "/nonexistent")
|
||||
self.stdout = stdout
|
||||
|
||||
def _run(self, args, timeout):
|
||||
return FakeProcess(self.stdout)
|
||||
|
||||
|
||||
def db_info(updated, version):
|
||||
return VersionRunner(json.dumps({"VulnerabilityDB": {"UpdatedAt": updated, "Version": version}})).db_info()
|
||||
|
||||
|
||||
check("db_info rejects an unparseable timestamp", db_info("yesterday", "2") is None)
|
||||
check("db_info rejects a non-string timestamp", db_info(5, "2") is None)
|
||||
check("db_info bounds the version", len(db_info("2026-11-09T06:13:52.123456789Z", "v" * 500)["version"]) == 128)
|
||||
check("mutation: db_info keeps a good database", db_info("2026-11-09T06:13:52Z", "2") == {"updatedAt": "2026-11-09T06:13:52Z", "version": "2"})
|
||||
check("a credential failure's reason is bounded", len(trivy.classify_failure("r" * 3000, "401 unauthorized").reason) <= 1024)
|
||||
|
||||
for url, want in (("https://a.test/x", "https://a.test/x"), ("http://a.test/", "http://a.test/"), ("HTTPS://a.test/x", ""),
|
||||
("HTTP://a.test/x", ""), ("Https://a.test/x", ""), ("javascript:alert(1)", ""), ("", "")):
|
||||
check("safe_url " + repr(url), trivy.safe_url(url) == want)
|
||||
check("every url safe_url keeps satisfies the report schema",
|
||||
report.re.search(report.SCHEMA["properties"]["findings"]["items"]["properties"]["url"]["pattern"], trivy.safe_url("https://a.test/x")))
|
||||
|
||||
# Recorded Trivy output through the normaliser: untrusted text.
|
||||
parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
|
||||
{"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y",
|
||||
|
||||
@@ -17,17 +17,48 @@ Hand-authored platform tooling (not a catalog service), reconciled by its own no
|
||||
The page is `cve.<domain>`, internal only, gated by the per-pod oauth2-proxy sidecar to `platform-admins`.
|
||||
The route is disowned from public DNS and the tunnel 404s it; PiHole answers it on the LAN.
|
||||
|
||||
## Operator step: add the Keycloak client (NOT done by the change that added this tree)
|
||||
## Operator action required: the Keycloak client (R-VULN-04 is NOT delivered until this is done)
|
||||
|
||||
This change adds everything around the Keycloak client but not the client. Adding it needs a new key in
|
||||
`platform/keycloak/keycloak-realm-clients.sops.yaml`, an encrypted file an agent must not decrypt; the
|
||||
L0 check `keycloak-client-secret-reconciler` then requires the client and that key to appear in the
|
||||
same change. Until this is done the page cannot complete a login (`invalid_client`, fail-closed); the
|
||||
Until these steps are done the page cannot complete a login (`invalid_client`, fail-closed). The
|
||||
scanner, its metrics and its alerts are unaffected.
|
||||
|
||||
In one PR (the realm import does not re-run on its own for a NEW client):
|
||||
An agent cannot do this, because every route runs through one encrypted file. The client's secret
|
||||
must be the key `vuln-scanner-client-secret` in `platform/keycloak/keycloak-realm-clients.sops.yaml`:
|
||||
the reconciler mounts only that Secret, and the L0 check `keycloak-client-secret-reconciler` rejects a
|
||||
placeholder that reads any other (a second source would be declared but never healed). That check
|
||||
also requires the client and the key to land in the same change. Adding a key needs the age private
|
||||
key. Separately, a completed realm import never re-runs, so a new client has to be created in the
|
||||
live realm once whatever git says. `*.sops.yaml` is a sensitive path, so the PR needs an approval or
|
||||
an operator-posted `/approve-override` (`infra/gitea/approval-policy.json`).
|
||||
|
||||
1. Generate one value and set it in both encrypted files, through pipes so it is never echoed:
|
||||
**Order matters.** Create the live client first: the reconciler treats every `*-client-secret` key as
|
||||
a client that must exist, and fails every 15-minute run with `NOT FOUND` otherwise
|
||||
(`KeycloakClientSecretReconcilerFailing`).
|
||||
|
||||
1. Create the client in the live realm. No secret is passed: the reconciler sets it from git after
|
||||
the merge, and it reads the admin credential from the pod's own environment, so nothing is echoed.
|
||||
|
||||
```bash
|
||||
kubectl --context k3s-prod -n keycloak exec keycloak-0 -- bash -c '
|
||||
K="/opt/keycloak/bin/kcadm.sh"; C="--config /tmp/kcadm-vuln.config"
|
||||
$K config credentials $C --server http://localhost:8080 --realm master \
|
||||
--user "$KC_BOOTSTRAP_ADMIN_USERNAME" --password "$KC_BOOTSTRAP_ADMIN_PASSWORD"
|
||||
D=home.keithjackson.dev
|
||||
ID=$($K create clients $C -r homelab -i -s clientId=vuln-scanner -s "name=CVE report (oauth2-proxy)" \
|
||||
-s protocol=openid-connect -s enabled=true -s publicClient=false -s standardFlowEnabled=true \
|
||||
-s directAccessGrantsEnabled=false -s "redirectUris=[\"https://cve.$D/oauth2/callback\"]" \
|
||||
-s "webOrigins=[\"https://cve.$D\"]")
|
||||
$K create clients/$ID/protocol-mappers/models $C -r homelab -s name=groups -s protocol=openid-connect \
|
||||
-s protocolMapper=oidc-group-membership-mapper -s "config.\"full.path\"=false" \
|
||||
-s "config.\"id.token.claim\"=true" -s "config.\"access.token.claim\"=true" \
|
||||
-s "config.\"userinfo.token.claim\"=true" -s "config.\"claim.name\"=groups"
|
||||
rm -f /tmp/kcadm-vuln.config'
|
||||
```
|
||||
|
||||
`D` is the prod `BASE_DOMAIN`; check it against the `cluster-vars` ConfigMap first.
|
||||
|
||||
2. On a branch, generate one value and set it in both encrypted files, through pipes so it is never
|
||||
echoed. `client-secret` in the scanner's file is replaced so that the two match:
|
||||
|
||||
```bash
|
||||
./infra/devcontainer/dev.sh run bash -c '
|
||||
@@ -38,7 +69,7 @@ In one PR (the realm import does not re-run on its own for a NEW client):
|
||||
"[\"stringData\"][\"client-secret\"]" "\"$cs\""'
|
||||
```
|
||||
|
||||
2. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client:
|
||||
3. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client:
|
||||
|
||||
```yaml
|
||||
# spec.placeholders
|
||||
@@ -72,9 +103,11 @@ In one PR (the realm import does not re-run on its own for a NEW client):
|
||||
"claim.name": groups
|
||||
```
|
||||
|
||||
3. Regenerate the derived URL file: `python3 ci/render-keycloak-client-urls.py --write`. Commit all
|
||||
three, gate, merge.
|
||||
4. Regenerate the derived URL file with `python3 ci/render-keycloak-client-urls.py --write`, and add
|
||||
`vuln-scanner-client-secret: REPLACE-ME` to `keycloak-realm-clients.sops.yaml.example`. Commit all of
|
||||
it in one PR, gate, merge.
|
||||
|
||||
4. After merge, the client must also exist in the live realm (a completed realm import does not re-run,
|
||||
per `platform/keycloak/README.md`, so create it once with `kcadm` or re-trigger the import). The
|
||||
client-secret reconciler then converges its secret and URLs.
|
||||
5. After Flux applies it, the next reconciler run sets the client's secret and URLs from git. Prove it
|
||||
with the L2 test `keycloak-client-secrets-match-git`, then log in at `https://cve.<domain>`. The
|
||||
`platform-admins` restriction is the sidecar's `--allowed-group`; the `groups` mapper above is what
|
||||
supplies the claim it reads, so a user outside that group gets a 403 from oauth2-proxy.
|
||||
@@ -133,6 +133,12 @@ spec:
|
||||
value: /tools/ca-certificates.crt
|
||||
- name: VULN_ACCEPTANCES
|
||||
value: /config/acceptances.yaml
|
||||
# Registries that need a credential this scanner deliberately does not hold (R-VULN-09).
|
||||
# Comma-separated hostnames. They report credential-required WITHOUT invoking Trivy:
|
||||
# the egress policy excludes the LAN, so a real attempt ends in a connection error and
|
||||
# would read as `failed`, firing VulnCoverageGap permanently. Flux substitutes the domain.
|
||||
- name: VULN_PRIVATE_REGISTRIES
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: PYTHONUNBUFFERED
|
||||
value: "1"
|
||||
- name: PYTHONDONTWRITEBYTECODE
|
||||
@@ -222,7 +228,7 @@ spec:
|
||||
env:
|
||||
# client-secret MUST equal the Keycloak `vuln-scanner` client's secret (key
|
||||
# vuln-scanner-client-secret in platform/keycloak/keycloak-realm-clients.sops.yaml; see
|
||||
# README.md, Operator step). Secrets by name only (R-IDP-04).
|
||||
# README.md, Operator action required). Secrets by name only (R-IDP-04).
|
||||
- name: OAUTH2_PROXY_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef: { name: vuln-scanner-oauth2-proxy, key: client-id }
|
||||
|
||||
@@ -15,6 +15,7 @@ from datetime import date, timedelta
|
||||
from state import utcnow
|
||||
|
||||
MAX_DAYS = 180
|
||||
MAX_REASON = 2048 # report.schema.json's limit; a longer reason would make the whole report invalid
|
||||
_KEYS = ("id", "image", "reason", "expires")
|
||||
_ENTRY = re.compile(r"^- ([a-z]+):(?: (.*))?$")
|
||||
_FIELD = re.compile(r"^ ([a-z]+):(?: (.*))?$")
|
||||
@@ -91,6 +92,8 @@ def _validate(fields, today):
|
||||
problems.append("id is missing or malformed")
|
||||
if not fields.get("reason"):
|
||||
problems.append("reason is mandatory")
|
||||
elif len(fields["reason"]) > MAX_REASON:
|
||||
problems.append("reason is longer than " + str(MAX_REASON) + " characters")
|
||||
if "image" in fields and not fields["image"]:
|
||||
problems.append("image is empty")
|
||||
expires = fields.get("expires") or ""
|
||||
|
||||
@@ -23,15 +23,21 @@ log = logging.getLogger("vuln-scanner")
|
||||
FULL_RUN_INTERVAL = timedelta(hours=20)
|
||||
DB_MAX_AGE = timedelta(hours=48)
|
||||
DB_REFRESH_AGE = timedelta(hours=24)
|
||||
KILLED_REASON = "scanner killed during scan (OOM/eviction?)"
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120):
|
||||
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120,
|
||||
private_registries=()):
|
||||
self.store = st.Store(data_dir)
|
||||
self.data_dir = data_dir
|
||||
self.client = client
|
||||
self.runner = runner
|
||||
self.acceptances_path = acceptances_path
|
||||
# Registries known to need credentials the scanner deliberately does not hold (R-VULN-09) and,
|
||||
# being on the LAN, not reachable from its egress policy: a scan would end in a connection
|
||||
# error (`failed`) rather than a 401 (`credential-required`), so it is never attempted.
|
||||
self.private_registries = {name.strip().lower() for name in private_registries if name.strip()}
|
||||
self.clock = clock
|
||||
self.tick_seconds = tick_seconds
|
||||
self.publish_seconds = publish_seconds
|
||||
@@ -43,6 +49,18 @@ class Engine:
|
||||
self.loop_thread = None
|
||||
self._last_publish = 0.0
|
||||
self._last_inventory = None
|
||||
self._recover_interrupted_scan()
|
||||
|
||||
def _recover_interrupted_scan(self):
|
||||
"""An inFlight key at startup means the last process died inside that scan."""
|
||||
key = self.state.get("inFlight")
|
||||
if not key:
|
||||
return
|
||||
log.error("the previous process was killed while scanning %s; marking it failed", key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is not None:
|
||||
record["killedAt"] = fmt_ts(self.clock())
|
||||
self._record_failure(key, "failed", KILLED_REASON)
|
||||
|
||||
# --- views used by the HTTP servers; none of them waits for a scan --------------------------
|
||||
def report_bytes(self):
|
||||
@@ -67,16 +85,19 @@ class Engine:
|
||||
|
||||
# --- persistence -----------------------------------------------------------------------------
|
||||
def publish(self):
|
||||
"""True when the report was written; False when it was refused and the previous one stands."""
|
||||
with self.lock:
|
||||
now = self.clock()
|
||||
entries, errors = acceptances.load(self.acceptances_path, now.date())
|
||||
for problem in errors:
|
||||
log.error("invalid acceptance ignored: %s", problem)
|
||||
built = report_mod.build(self.state, entries, errors, now)
|
||||
if report_mod.publish(built, self.data_dir):
|
||||
published = report_mod.publish(built, self.data_dir)
|
||||
if published:
|
||||
self.report = built
|
||||
self.report_json = report_mod.serialise(built)
|
||||
self._last_publish = time.monotonic()
|
||||
return published
|
||||
|
||||
# --- inventory -------------------------------------------------------------------------------
|
||||
def refresh_inventory(self):
|
||||
@@ -152,6 +173,14 @@ class Engine:
|
||||
return timedelta.max if moment is None else self.clock() - moment
|
||||
|
||||
def _scan_unit(self, key, reference):
|
||||
registry = trivy.registry_of(reference)
|
||||
if registry in self.private_registries or registry.split(":", 1)[0] in self.private_registries:
|
||||
self._record_failure(key, "credential-required",
|
||||
"registry " + registry + " is private and the scanner holds no credential for it")
|
||||
return
|
||||
with self.lock:
|
||||
self.state["inFlight"] = key
|
||||
self.store.save(self.state)
|
||||
try:
|
||||
found = trivy.parse_results(self.runner.scan(reference))
|
||||
except trivy.ScanFailure as exc:
|
||||
@@ -166,11 +195,24 @@ class Engine:
|
||||
except Exception: # pylint: disable=broad-except
|
||||
log.exception("scan cache cleanup failed")
|
||||
|
||||
def _finish_in_flight(self, key):
|
||||
"""Cleared in the same write as the scan's outcome, so no crash can separate the two."""
|
||||
if self.state.get("inFlight") == key:
|
||||
self.state["inFlight"] = None
|
||||
|
||||
def _killed_recently(self, key):
|
||||
"""A unit that killed the scanner sits out full runs for a day: retrying it first on every
|
||||
restart would OOM again before any later image, which is the wedge this exists to prevent."""
|
||||
killed = parse_ts(self.state["scans"][key].get("killedAt"))
|
||||
return killed is not None and self.clock() - killed < FULL_RUN_INTERVAL
|
||||
|
||||
def _record_failure(self, key, kind, reason):
|
||||
now = fmt_ts(self.clock())
|
||||
with self.lock:
|
||||
self._finish_in_flight(key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is None:
|
||||
self.store.save(self.state)
|
||||
return
|
||||
record["lastAttemptAt"] = now
|
||||
record["outcome"] = kind
|
||||
@@ -181,10 +223,13 @@ class Engine:
|
||||
def _record_success(self, key, found):
|
||||
now = fmt_ts(self.clock())
|
||||
with self.lock:
|
||||
self._finish_in_flight(key)
|
||||
record = self.state["scans"].get(key)
|
||||
if record is None:
|
||||
self.store.save(self.state)
|
||||
return
|
||||
record["lastAttemptAt"] = now
|
||||
record["killedAt"] = None
|
||||
record["lastSuccessAt"] = now
|
||||
record["outcome"] = "ok"
|
||||
record["reason"] = None
|
||||
@@ -215,7 +260,10 @@ class Engine:
|
||||
started = self.clock()
|
||||
log.info("full run starting")
|
||||
database_ok = self._refresh_database(always=True)
|
||||
self._scan_queue(sorted(self._units()))
|
||||
self._scan_queue([key for key in sorted(self._units()) if not self._killed_recently(key)])
|
||||
if self.stop.is_set():
|
||||
log.info("full run interrupted by shutdown; not recorded")
|
||||
return
|
||||
info = self.state["database"]
|
||||
now = self.clock()
|
||||
degraded = not database_ok or info is None or self._db_age(info) > DB_MAX_AGE
|
||||
@@ -225,8 +273,15 @@ class Engine:
|
||||
status = "degraded" if degraded else "complete"
|
||||
with self.lock:
|
||||
self.state["run"] = {"startedAt": fmt_ts(started), "finishedAt": fmt_ts(now), "status": status}
|
||||
if not degraded:
|
||||
self.store.save(self.state)
|
||||
# Only a report that actually reached the store counts as a completed run. Otherwise a
|
||||
# refused report would freeze the page while VulnScanStale stayed quiet.
|
||||
if self.publish() and not degraded and self.report["run"]["status"] == "complete":
|
||||
self.state["lastCompleteRunAt"] = fmt_ts(now)
|
||||
# A corrupt state.json degrades the run that rebuilds from it, and no further: once a full
|
||||
# run has completed against a good inventory, the state is whole again.
|
||||
if self.state.get("loadError") and self._last_inventory is not None and not self.state.get("inventoryError"):
|
||||
self.state["loadError"] = None
|
||||
self.store.save(self.state)
|
||||
log.info("full run finished: %s", status)
|
||||
|
||||
@@ -262,7 +317,9 @@ def main():
|
||||
tools_dir = os.environ.get("VULN_TOOLS_DIR", "/tools")
|
||||
trivy_bin = os.environ.get("TRIVY_BIN", os.path.join(tools_dir, "trivy"))
|
||||
runner = trivy.Runner(trivy_bin, cache_dir, int(os.environ.get("VULN_SCAN_TIMEOUT_SECONDS", "600")))
|
||||
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"))
|
||||
private_registries = os.environ.get("VULN_PRIVATE_REGISTRIES", "").split(",")
|
||||
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"),
|
||||
private_registries=private_registries)
|
||||
server.start_all(engine, ("127.0.0.1", 8080), ("0.0.0.0", 8081), ("0.0.0.0", 9090))
|
||||
finished = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: finished.set())
|
||||
|
||||
@@ -180,14 +180,39 @@ def _sev(severity):
|
||||
return '<span class="sev-' + e(severity) + '">' + e(severity) + "</span>"
|
||||
|
||||
|
||||
def _workload_lines(target):
|
||||
def _state_label(target):
|
||||
return "target " + e(target["state"]) + (" (" + e(target["reason"]) + ")" if target.get("reason") else "")
|
||||
|
||||
|
||||
def _status_label(finding):
|
||||
"""One finding's own status, with its own acceptance: never another image's."""
|
||||
label = "status " + e(finding["status"])
|
||||
acceptance = finding.get("acceptance")
|
||||
if acceptance:
|
||||
label += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
|
||||
return label
|
||||
|
||||
|
||||
def _workload_lines(target, finding=None):
|
||||
"""One line per workload, each carrying the target state and, in a CVE expansion, the status."""
|
||||
suffix = " — " + _state_label(target) + (" — " + _status_label(finding) if finding else "")
|
||||
items = []
|
||||
for workload in target["workloads"]:
|
||||
items.append("<li>" + e(workload["namespace"]) + " / " + e(workload["kind"]) + " / " + e(workload["name"])
|
||||
+ " — containers: " + e(", ".join(workload["containers"]) or "none") + "</li>")
|
||||
+ " — containers: " + e(", ".join(workload["containers"]) or "none") + suffix + "</li>")
|
||||
return "<ul>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def _status_summary(statuses):
|
||||
"""The row-level status. A CVE accepted on one image and open on another reads as mixed."""
|
||||
counts = {}
|
||||
for status in statuses:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
if len(counts) == 1:
|
||||
return e(next(iter(counts)))
|
||||
return "mixed: " + ", ".join(e(counts[s]) + " " + e(s) for s in sorted(counts, key=STATUS_RANK.get))
|
||||
|
||||
|
||||
def _remediation_cell(images, remediation):
|
||||
if not images:
|
||||
return ""
|
||||
@@ -250,13 +275,10 @@ def _cves(report, targets, params, remediation):
|
||||
for finding, target in row["items"]:
|
||||
if finding["status"] == "fix-available":
|
||||
fix_images.add(target["image"])
|
||||
lines.append("<li>" + e(target["id"]) + " [" + e(target["state"]) + "] — " + e(finding["package"]) + " "
|
||||
+ e(finding["installedVersion"]) + " → " + e(finding["fixedVersion"] or "no fix")
|
||||
+ _workload_lines(target) + "</li>")
|
||||
acceptance = next((f.get("acceptance") for f, _ in row["items"] if f.get("acceptance")), None)
|
||||
status_text = e(row["status"])
|
||||
if acceptance:
|
||||
status_text += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
|
||||
lines.append("<li>" + e(target["id"]) + " — " + _state_label(target) + " — " + _status_label(finding)
|
||||
+ " — " + e(finding["package"]) + " " + e(finding["installedVersion"]) + " → "
|
||||
+ e(finding["fixedVersion"] or "no fix") + _workload_lines(target, finding) + "</li>")
|
||||
status_text = _status_summary(f["status"] for f, _ in row["items"])
|
||||
html_rows.append(
|
||||
"<tr><td>" + _link(row["url"], row["id"]) + "</td><td>" + _sev(row["severity"]) + "</td><td>"
|
||||
+ e("" if row["cvss"] is None else row["cvss"]) + "</td><td>" + e(row["package"]) + "</td><td>" + e(row["fix"])
|
||||
@@ -317,12 +339,13 @@ def _workloads(report, targets, params):
|
||||
html_rows = []
|
||||
for row in page_rows:
|
||||
lines = "".join(
|
||||
"<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + e(f["status"]) + "</li>"
|
||||
for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"])))
|
||||
"<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + _status_label(f)
|
||||
+ " — " + e(f["targetId"]) + " — " + _state_label(targets[f["targetId"]]) + "</li>"
|
||||
for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"], f["targetId"])))
|
||||
counts = "".join("<td>" + e(row["counts"][s]) + "</td>" for s in SEVERITIES)
|
||||
html_rows.append(
|
||||
"<tr><td>" + e(row["key"][0]) + "</td><td>" + e(row["key"][1]) + "</td><td>" + e(row["key"][2]) + "</td><td>"
|
||||
+ e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) for i in sorted(row["images"])) + "</td>"
|
||||
+ e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) + " — " + _state_label(targets[i]) for i in sorted(row["images"])) + "</td>"
|
||||
+ counts + "<td><details><summary>" + e(len(row["findings"])) + " findings</summary><ul>" + lines + "</ul></details></td></tr>")
|
||||
head = [_th(params, "Namespace", "namespace"), _th(params, "Kind", "kind"), _th(params, "Workload", "name"), "<th>Containers</th>",
|
||||
_th(params, "Images", "images")] + [_th(params, s, s.lower()) for s in SEVERITIES] + [_th(params, "Findings", "total")]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Build, validate and publish report.json, the one interface to the page and the bot."""
|
||||
import json
|
||||
import logging
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
@@ -85,10 +86,15 @@ def build(state, accepted, acceptance_errors, now):
|
||||
if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
|
||||
or db_time is None or now - db_time > DB_STALE:
|
||||
run["status"] = "degraded"
|
||||
return {
|
||||
built = {
|
||||
"schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
|
||||
"targets": targets, "findings": findings, "remediation": list(state["remediation"]),
|
||||
}
|
||||
repaired = _isolate(built)
|
||||
if repaired:
|
||||
log.error("%d report entries held a value the schema refuses; repaired or dropped them", repaired)
|
||||
run["status"] = "degraded"
|
||||
return built
|
||||
|
||||
|
||||
# --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
|
||||
@@ -175,6 +181,88 @@ def _check(schema, value, path, errors):
|
||||
_check(branch, value, path, errors)
|
||||
|
||||
|
||||
# --- isolation: one bad upstream value costs that value, never the whole report ---------------------
|
||||
|
||||
def _repair(schema, value):
|
||||
"""(usable, value): value itself when valid, else the nearest valid form of it."""
|
||||
trial = []
|
||||
_check(schema, value, "$", trial)
|
||||
if not trial:
|
||||
return True, value
|
||||
if isinstance(value, dict) and "properties" in schema:
|
||||
fixed = {}
|
||||
for key, sub in schema["properties"].items():
|
||||
if key not in value:
|
||||
continue
|
||||
usable, item = _repair(sub, value[key])
|
||||
if usable:
|
||||
fixed[key] = item
|
||||
elif key in schema.get("required", []):
|
||||
return False, None
|
||||
trial = []
|
||||
_check(schema, fixed, "$", trial)
|
||||
return not trial, fixed
|
||||
if isinstance(value, list) and "items" in schema:
|
||||
repaired = [_repair(schema["items"], item) for item in value]
|
||||
return True, [item for usable, item in repaired if usable]
|
||||
candidates = [None, "", "UNKNOWN"]
|
||||
if isinstance(value, str) and "maxLength" in schema:
|
||||
candidates.insert(0, value[:schema["maxLength"]])
|
||||
for candidate in candidates:
|
||||
trial = []
|
||||
_check(schema, candidate, "$", trial)
|
||||
if not trial:
|
||||
return True, candidate
|
||||
return False, None
|
||||
|
||||
|
||||
def _isolate(report):
|
||||
"""Repair (or, as a last resort, drop) each entry the schema refuses. Returns how many changed."""
|
||||
changed = 0
|
||||
schema = SCHEMA["properties"]
|
||||
usable, database = _repair(schema["database"], report["database"])
|
||||
if database is not report["database"]:
|
||||
changed += 1
|
||||
report["database"] = database if usable else {"updatedAt": EPOCH, "version": "unavailable"}
|
||||
renamed, dropped, targets = {}, set(), []
|
||||
for target in report["targets"]:
|
||||
original = target["id"]
|
||||
if len(original) > 2048: # keep it unique: a truncated id may not collide with another
|
||||
target = dict(target, id=original[:2031] + "#" + hashlib.sha256(original.encode("utf-8")).hexdigest()[:16])
|
||||
renamed[original] = target["id"]
|
||||
usable, fixed = _repair(schema["targets"]["items"], target)
|
||||
if not usable:
|
||||
changed += 1
|
||||
dropped.add(original)
|
||||
continue
|
||||
if fixed is not target or original in renamed:
|
||||
changed += 1
|
||||
targets.append(fixed)
|
||||
report["targets"] = targets
|
||||
findings = []
|
||||
for finding in report["findings"]:
|
||||
if finding["targetId"] in dropped:
|
||||
changed += 1
|
||||
continue
|
||||
if finding["targetId"] in renamed:
|
||||
finding = dict(finding, targetId=renamed[finding["targetId"]])
|
||||
usable, fixed = _repair(schema["findings"]["items"], finding)
|
||||
if not usable or fixed is not finding:
|
||||
changed += 1
|
||||
if usable:
|
||||
findings.append(fixed)
|
||||
report["findings"] = findings
|
||||
remediation = []
|
||||
for entry in report["remediation"]:
|
||||
usable, fixed = _repair(schema["remediation"]["items"], entry)
|
||||
if not usable or fixed is not entry:
|
||||
changed += 1
|
||||
if usable:
|
||||
remediation.append(fixed)
|
||||
report["remediation"] = remediation
|
||||
return changed
|
||||
|
||||
|
||||
def validate(report):
|
||||
"""Schema and referential errors; an empty list means the report may be served."""
|
||||
errors = []
|
||||
|
||||
@@ -126,7 +126,9 @@ def validate_triage(document):
|
||||
if entry["state"] not in REMEDIATION_STATES:
|
||||
return [prefix + "unknown state"]
|
||||
url = entry["url"]
|
||||
if url is not None and (not isinstance(url, str) or len(url) > 2048 or trivy.safe_url(url) != url):
|
||||
# safe_url is the same rule the report schema enforces, so a value accepted here cannot make
|
||||
# the report invalid. Empty is refused: safe_url returns "" for it, and the schema does not.
|
||||
if url is not None and (not isinstance(url, str) or not url or trivy.safe_url(url) != url):
|
||||
return [prefix + "url must be null or an http(s) URL"]
|
||||
if parse_ts(entry["updatedAt"]) is None or len(entry["updatedAt"]) > 64:
|
||||
return [prefix + "updatedAt must be RFC 3339"]
|
||||
|
||||
@@ -72,6 +72,9 @@ def new_state():
|
||||
"remediation": [],
|
||||
"triageHeartbeatAt": None,
|
||||
"loadError": None,
|
||||
# The scan key Trivy is running against right now, persisted BEFORE the scan starts: an OOM
|
||||
# kill takes the whole container, so nothing can record the failure afterwards.
|
||||
"inFlight": None,
|
||||
}
|
||||
|
||||
|
||||
@@ -84,6 +87,7 @@ def new_record(now):
|
||||
"outcome": None,
|
||||
"reason": None,
|
||||
"failedSince": None,
|
||||
"killedAt": None,
|
||||
"findings": [],
|
||||
"firstSeen": {},
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import re
|
||||
import subprocess
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from state import parse_ts
|
||||
|
||||
log = logging.getLogger("vuln-scanner.trivy")
|
||||
|
||||
SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
|
||||
@@ -13,7 +15,11 @@ _CREDENTIAL = re.compile(
|
||||
r"|status(?: code)?:? ?(?:401|403)\b|\b40[13] (?:unauthorized|forbidden)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
# Lone surrogates too: json.loads accepts them, and they cannot be encoded as UTF-8 for the page.
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f\ud800-\udfff]")
|
||||
# Exact and lowercase, matching report.schema.json, whose pattern is case-sensitive: an `HTTPS://x`
|
||||
# accepted here would make the schema refuse the entire report.
|
||||
_URL_SCHEME = re.compile(r"https?://")
|
||||
|
||||
|
||||
class ScanFailure(Exception):
|
||||
@@ -35,13 +41,18 @@ def text(value, limit=512):
|
||||
def safe_url(value):
|
||||
"""value if it is a plain http(s) URL, else the empty string."""
|
||||
url = text(value, 2048)
|
||||
if not url or url != value or re.search(r"\s", url):
|
||||
if not url or url != value or re.search(r"\s", url) or not _URL_SCHEME.match(url):
|
||||
return ""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
except ValueError:
|
||||
return ""
|
||||
return url if parts.scheme.lower() in ("http", "https") and parts.netloc else ""
|
||||
return url if parts.netloc else ""
|
||||
|
||||
|
||||
def registry_of(reference):
|
||||
"""The lowercase registry host (and port) of a normalised registry/repository reference."""
|
||||
return reference.split("/", 1)[0].lower()
|
||||
|
||||
|
||||
def _cvss(vuln):
|
||||
@@ -95,7 +106,7 @@ def parse_results(document):
|
||||
def classify_failure(registry, stderr):
|
||||
"""Sort a non-zero Trivy exit into credential-required or failed. Reasons name the registry."""
|
||||
if _CREDENTIAL.search(stderr or ""):
|
||||
return ScanFailure("credential-required", "registry " + registry + " refused anonymous access")
|
||||
return ScanFailure("credential-required", "registry " + text(registry, 253) + " refused anonymous access")
|
||||
lines = [line.strip() for line in (stderr or "").splitlines() if line.strip()]
|
||||
return ScanFailure("failed", text(lines[-1], 400) if lines else "trivy exited with an error and no message")
|
||||
|
||||
@@ -125,9 +136,9 @@ class Runner:
|
||||
db = json.loads(result.stdout).get("VulnerabilityDB")
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
if not isinstance(db, dict) or not db.get("UpdatedAt"):
|
||||
return None
|
||||
return {"updatedAt": db["UpdatedAt"], "version": str(db.get("Version", "unknown"))}
|
||||
if not isinstance(db, dict) or parse_ts(db.get("UpdatedAt")) is None:
|
||||
return None # an unparseable timestamp is no usable database, and must not reach the report
|
||||
return {"updatedAt": db["UpdatedAt"], "version": text(str(db.get("Version", "unknown")), 128) or "unknown"}
|
||||
|
||||
def scan(self, reference):
|
||||
args = [
|
||||
@@ -143,7 +154,7 @@ class Runner:
|
||||
if result.returncode < 0:
|
||||
raise ScanFailure("failed", "trivy killed by signal " + str(-result.returncode) + " (out of memory?)")
|
||||
if result.returncode != 0:
|
||||
raise classify_failure(reference.split("/", 1)[0], result.stderr)
|
||||
raise classify_failure(registry_of(reference), result.stderr)
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except ValueError:
|
||||
|
||||
@@ -483,6 +483,16 @@
|
||||
"description": "ADDED 2026-10-30. MAJORS WAIT FOR A TICK ON THE DEPENDENCY DASHBOARD instead of opening a PR. They are never auto-merged anyway, and with prConcurrentLimit 5 the dry run's 10 majors would hold every PR slot open indefinitely, starving the patch/minor/digest PRs that can merge on green.",
|
||||
"matchUpdateTypes": ["major"],
|
||||
"dependencyDashboardApproval": true
|
||||
},
|
||||
// ADDED 2026-11-09 — EVERY IMAGE IN THE SCANNER POD, NOT ONLY TRIVY. The python image runs the
|
||||
// process that holds the cluster-read token (rbac.yaml, R-VULN-09), and without this rule its
|
||||
// digest and patch bumps fall through to the auto-merge rule above. LAST ON PURPOSE: Renovate
|
||||
// applies packageRules in order and a later match overrides, so nothing above can undo it.
|
||||
{
|
||||
"description": "Anything under platform/vuln-scanner: the python image holds the cluster-read token. Never auto-merged, 7-day minimum release age (R-VULN-09).",
|
||||
"matchFileNames": ["platform/vuln-scanner/**"],
|
||||
"automerge": false,
|
||||
"minimumReleaseAge": "7 days"
|
||||
}
|
||||
],
|
||||
// WHAT RENOVATE STILL CANNOT FIX, so nobody mistakes a working bot for full coverage. A
|
||||
|
||||
Reference in new issue
Block a user