feat(vuln): automated CVE visibility for production (R-VULN-*, ADR-0027) #165

Open
gitea-admin wants to merge 15 commits from feat/vuln-visibility into main
pull from: feat/vuln-visibility
15 changed files with 557 additions and 43 deletions
Showing only changes of commit d8b25a67fe - Show all commits

No files matched your search

@@ -16,6 +16,11 @@
# (6) the pod: non-root, read-only rootfs, drop ALL, seccomp RuntimeDefault, no host
# namespaces or hostPath, CPU and memory limits on every container, an ephemeral-storage
# limit on the scanner, and a size-limited /cache emptyDir.
# (8) a renovate.json5 rule covers every file under platform/vuln-scanner: automerge:false,
# minimumReleaseAge of at least 7 days, and no later rule turns auto-merge back on (the
# python image holds the cluster-read token);
# (9) the scanner names the private registry it cannot reach, so that registry reports
# credential-required rather than a permanent failed.
# AND (7) each evaluator is run on a mutated copy and must flag it (non-vacuity).
# =============================================================================
set -euo pipefail
@@ -150,6 +155,33 @@ def eval_renovate(rules):
return bad
SCANNER_FILES = "platform/vuln-scanner/**"
def eval_renovate_files(rules):
positions = [i for i, r in enumerate(rules) if SCANNER_FILES in (r.get("matchFileNames") or [])]
if not positions:
return [f"no packageRule has matchFileNames {SCANNER_FILES}"]
bad = []
rule = rules[positions[-1]]
if rule.get("automerge") is not False:
bad.append("the platform/vuln-scanner file rule is not automerge:false")
age = VERSION_UPDATE_RULE_RE.match(rule.get("minimumReleaseAge") or "")
if not age or int(age.group(1)) < 7:
bad.append("the platform/vuln-scanner file rule has no minimumReleaseAge of at least 7 days")
if any(r.get("automerge") is True for r in rules[positions[-1] + 1:]):
bad.append("a rule after the platform/vuln-scanner file rule re-enables auto-merge")
return bad
def eval_private_registries(pod):
scanner = next(c for c in pod["containers"] if c["name"] == "scanner")
env = {e["name"]: e.get("value") for e in scanner.get("env") or []}
if env.get("VULN_PRIVATE_REGISTRIES") != "gitea.${BASE_DOMAIN}":
return ["the scanner does not name the private registry gitea.${BASE_DOMAIN}"]
return []
def eval_rbac(rbac):
bad = []
croles = [d for d in rbac if d.get("kind") == "ClusterRole"]
@@ -224,6 +256,10 @@ rules = renovate.get("packageRules") or []
check(len(rules) > 5, "(4) parsed renovate.json5 packageRules (parser degenerated)")
for v in eval_renovate(rules):
check(False, f"(4) {v}")
for v in eval_renovate_files(rules):
check(False, f"(8) {v}")
for v in eval_private_registries(pod):
check(False, f"(9) {v}")
rbac = docs(COMP / "rbac.yaml")
for v in eval_rbac(rbac):
@@ -274,6 +310,23 @@ for name, fn in (("Trivy automerge flipped true", ren_auto), ("minimumReleaseAge
("Trivy removed from Renovate rules", ren_drop), ("later auto-merge rule appended", ren_later)):
check(bool(eval_renovate(mutated(fn, rules))), f"(7) Renovate evaluator missed mutant: {name}")
# Renovate file-rule mutants
def files_rule(r):
return next(x for x in r if SCANNER_FILES in (x.get("matchFileNames") or []))
def files_auto(r): files_rule(r)["automerge"] = True
def files_age(r): files_rule(r)["minimumReleaseAge"] = "1 day"
def files_drop(r): r.remove(files_rule(r))
def files_later(r): r.append({"matchDatasources": ["docker"], "automerge": True})
for name, fn in (("file rule automerge true", files_auto), ("file rule age shortened", files_age),
("file rule removed", files_drop), ("later rule re-enables auto-merge", files_later)):
check(bool(eval_renovate_files(mutated(fn, rules))), f"(7) file-rule evaluator missed mutant: {name}")
def reg_drop(p):
next(c for c in p["containers"] if c["name"] == "scanner")["env"][:] = [
e for e in next(c for c in p["containers"] if c["name"] == "scanner")["env"]
if e["name"] != "VULN_PRIVATE_REGISTRIES"]
check(bool(eval_private_registries(mutated(reg_drop, pod))), "(7) private-registry evaluator missed the removal")
# Pod mutants
def pod_root(p): p["securityContext"]["runAsNonRoot"] = False
def pod_host(p): p["hostNetwork"] = True
@@ -94,6 +94,15 @@ def status(when, accepted, errs=()):
return built, built["findings"][0]
def reason_entry(length):
return acceptances.parse_text("- id: CVE-1\n reason: " + "r" * length + "\n expires: 2026-12-01\n", NOW.date())
found, problems = reason_entry(3000)
check("a reason over 2048 characters ignores the entry and reports it", found == [] and any("2048" in p for p in problems))
found, problems = reason_entry(2048)
check("mutation: a reason of exactly 2048 characters is valid", len(found) == 1 and problems == [])
entry = {"id": "CVE-1", "image": None, "reason": "not reachable", "expires": date(2026, 11, 9)}
built, finding = status(NOW, [entry])
check("unexpired acceptance marks the finding accepted", finding["status"] == "accepted"
@@ -150,6 +150,10 @@ with tempfile.TemporaryDirectory() as tmp:
mutate(document)
status, _, _ = post(internal, document)
check("triage rejects " + name, status == 400)
for name, url in (("an uppercase scheme", "HTTP://gitea.test/pulls/1"), ("an empty url", ""), ("a javascript url", "javascript:alert(1)")):
bad = json.loads(json.dumps(valid))
bad["remediation"][0]["url"] = url
check("triage rejects " + name, post(internal, bad)[0] == 400)
check("triage rejects a wrong content type", post(internal, valid, "text/plain")[0] == 415)
status, _, _ = request(internal, "POST", "/triage", b"x" * 300000, {"Content-Type": "application/json"})
check("triage rejects an oversized body", status == 413)
@@ -170,6 +170,39 @@ _, _, body = render()
check("default view excludes nothing", all(f["id"] in body for f in REPORT["findings"]))
check("remediation column shows pr-open with a link", 'href="https://gitea.test/pulls/1"' in body and "pr-open" in body)
# Status and state belong to the line they describe (R-VULN-04).
images = [t for t in REPORT["targets"] if t["type"] == "image" and t["workloads"]]
check("fixture has two image targets with workloads", len(images) >= 2)
mixed = copy.deepcopy(REPORT)
first, second = next(t for t in mixed["targets"] if t["id"] == images[0]["id"]), next(t for t in mixed["targets"] if t["id"] == images[1]["id"])
first["state"], first["reason"] = "scanned", None
second["state"], second["reason"] = "stale", "last success long ago"
base = dict(REPORT["findings"][0], id="CVE-MIXED")
base.pop("acceptance", None)
mixed["findings"] = [
dict(base, targetId=first["id"], status="accepted", acceptance={"reason": "only-on-the-first", "expires": "2026-12-01"}),
dict(base, targetId=second["id"], status="fix-available"),
]
_, _, body = render(mixed, "view=cves&q=CVE-MIXED")
check("a mixed CVE reads as mixed, with counts", "mixed: 1 fix-available, 1 accepted" in body)
check("a mixed CVE's row does not carry one image's acceptance", "<td>mixed: 1 fix-available, 1 accepted</td>" in body)
first_line = page.e(first["id"]) + " — target scanned — status accepted (only-on-the-first, until 2026-12-01)"
second_line = page.e(second["id"]) + " — target stale (last success long ago) — status fix-available"
check("the accepted image's line carries its own acceptance", first_line in body)
check("the other image's line carries its own status and state", second_line in body)
check("no line attaches the acceptance to the unaccepted status", "status fix-available (only-on-the-first" not in body)
check("workload lines inside the expansion carry state and status",
"target stale (last success long ago) — status fix-available</li>" in body
and "containers: " + page.e(", ".join(second["workloads"][0]["containers"]) or "none") + " — target stale" in body)
_, _, plain = render(query="view=cves&status=accepted")
check("mutation: a uniformly accepted CVE does not read as mixed", "mixed:" not in plain and "CVE-2026-1003" in plain)
_, _, body = render(mixed, "view=workloads")
check("the workloads images column labels each image's state", page.e(second["id"]) + " — target stale (last success long ago)" in body)
check("the workloads expansion labels each finding's status, image and state",
"status fix-available — " + page.e(second["id"]) + " — target stale (last success long ago)</li>" in body)
check("the workloads expansion keeps an acceptance on its own line",
"status accepted (only-on-the-first, until 2026-12-01) — " + page.e(first["id"]) + " — target scanned</li>" in body)
big = copy.deepcopy(REPORT)
big["findings"] = [dict(REPORT["findings"][0], id="CVE-2026-" + format(i, "05d")) for i in range(250)]
_, _, body = render(big, "view=cves&size=50&page=2")
@@ -95,14 +95,14 @@ class FakeRunner:
pass
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER):
def engine(tmp, runner=None, clock=None, acc="[]\n", cluster=CLUSTER, private=()):
path = os.path.join(tmp, "acceptances.yaml")
if not os.path.exists(path):
Path(path).write_text(acc)
clock = clock or Clock()
runner = runner or FakeRunner()
runner.clock = clock
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0), runner, clock
return main.Engine(tmp, FakeClient(cluster), runner, path, clock=clock, publish_seconds=0, private_registries=private), runner, clock
def targets(eng):
@@ -239,6 +239,184 @@ with tempfile.TemporaryDirectory() as tmp:
thread.join(10)
check("a crash in one cycle is logged and the loop continues", len(calls) == 2 and not thread.is_alive())
class Killed(BaseException):
"""What an OOM kill looks like from inside the process: no `except Exception` runs."""
class KillingRunner(FakeRunner):
def __init__(self, victim, data_dir):
super().__init__()
self.victim = victim
self.data_dir = data_dir
self.on_disk = None
def scan(self, reference):
if reference == self.victim:
self.on_disk = json.loads(Path(self.data_dir, "state.json").read_text()).get("inFlight")
raise Killed()
return super().scan(reference)
with tempfile.TemporaryDirectory() as tmp:
# R-VULN-05 BLOCKER: a scan that kills the container must cost that image only.
eng, runner, clock = engine(tmp, runner=KillingRunner(POSTGRES, tmp))
killed = False
try:
eng.tick()
except Killed:
killed = True
check("kill: the simulated OOM really aborted the process path", killed)
check("kill: inFlight is on disk BEFORE the scan starts", runner.on_disk == D4)
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
check("kill: startup clears inFlight", eng2.state["inFlight"] is None)
check("kill: clearing is persisted", json.loads(Path(tmp, "state.json").read_text())["inFlight"] is None)
eng2.publish()
t = targets(eng2)
check("kill: the target is failed", t[POSTGRES]["state"] == "failed")
check("kill: the reason is the killed-scan text", t[POSTGRES]["reason"] == main.KILLED_REASON)
eng2.tick()
t = targets(eng2)
check("kill: images after the killed one get scanned", PRIVATE in runner2.scans and t[PRIVATE]["state"] == "credential-required")
check("kill: the killed image is not retried on restart (no wedge)", POSTGRES not in runner2.scans)
check("kill: the killed target stays failed with its reason", t[POSTGRES]["reason"] == main.KILLED_REASON)
check("kill: inFlight is clear after a normal cycle", eng2.state["inFlight"] is None)
clock.advance(hours=21)
eng2.tick()
check("kill: the killed image is retried after the sit-out", runner2.scans.count(POSTGRES) == 1)
check("kill: a successful retry clears the failure", targets(eng2)[POSTGRES]["state"] == "scanned")
check("kill: a successful retry clears killedAt", eng2.state["scans"][D4]["killedAt"] is None)
with tempfile.TemporaryDirectory() as tmp:
eng, runner, clock = engine(tmp)
eng.refresh_inventory()
eng2, runner2, clock2 = engine(tmp, runner=FakeRunner(), clock=clock)
eng2.publish()
check("mutation: a restart with no inFlight fails nothing", targets(eng2)[POSTGRES]["state"] == "pending")
with tempfile.TemporaryDirectory() as tmp:
# A registry the scanner cannot reach is credential-required, and Trivy is never invoked.
eng, runner, clock = engine(tmp, private=[" GHCR.io ", ""])
eng.tick()
t = targets(eng)
check("private registry: reported credential-required", t[PRIVATE]["state"] == "credential-required")
check("private registry: the reason names the registry", "ghcr.io" in t[PRIVATE]["reason"])
check("private registry: Trivy is never invoked for it", PRIVATE not in runner.scans and NIGHTLY not in runner.scans)
check("private registry: other registries still scan", NGINX in runner.scans)
check("private registry: not counted as failed", report.failed_over_24h(eng.state, clock() + timedelta(hours=48)) == 0)
with tempfile.TemporaryDirectory() as tmp:
eng, runner, clock = engine(tmp)
eng.tick()
check("mutation: with no private registry configured Trivy is invoked", PRIVATE in runner.scans)
with tempfile.TemporaryDirectory() as tmp:
# A corrupt state.json degrades the run that rebuilds from it, not every run thereafter.
Path(tmp, "state.json").write_text("{not json")
eng, runner, clock = engine(tmp)
check("loadError: set after a corrupt state.json", eng.state["loadError"])
eng.tick()
check("loadError: the rebuilding run is degraded", eng.report["run"]["status"] == "degraded")
check("loadError: cleared after a full run with a good inventory", eng.state["loadError"] is None)
clock.advance(hours=21)
eng.tick()
check("loadError: the next full run is complete", eng.report["run"]["status"] == "complete" and eng.state["lastCompleteRunAt"])
with tempfile.TemporaryDirectory() as tmp:
Path(tmp, "state.json").write_text("{not json")
eng, runner, clock = engine(tmp)
eng.client.fail = True
eng.tick()
check("mutation: loadError is kept when the inventory failed", eng.state["loadError"])
with tempfile.TemporaryDirectory() as tmp:
# lastCompleteRunAt means a report reached the store; a refused report must not advance it.
eng, runner, clock = engine(tmp)
real_publish = report.publish
report.publish = lambda *_: False
try:
eng.tick()
finally:
report.publish = real_publish
check("a refused publish does not advance lastCompleteRunAt", eng.state["lastCompleteRunAt"] is None)
clock.advance(hours=21)
eng.tick()
check("mutation: a successful publish does advance it", eng.state["lastCompleteRunAt"] is not None)
with tempfile.TemporaryDirectory() as tmp:
# One upstream string costs that string. Each of these used to freeze the whole report.
long_reason = "x" * 3000
eng, runner, clock = engine(tmp, acc="- id: CVE-2026-0010\n reason: " + long_reason + "\n expires: 2026-12-01\n")
runner.outcomes[NGINX] = {"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
{"VulnerabilityID": "CVE-UP", "Severity": "HIGH", "PkgName": "p", "InstalledVersion": "1",
"PrimaryURL": "HTTPS://upper.example/x", "Title": "t\ud800"}]}]}
eng.tick()
check("hostile strings: the report still publishes", json.loads(Path(tmp, "report.json").read_text())["findings"] and report.validate(eng.report) == [])
up = [f for f in findings(eng, NGINX) if f["id"] == "CVE-UP"][0]
check("hostile strings: an uppercase scheme drops only the URL", up["url"] == "")
check("hostile strings: a lone surrogate is removed", "\ud800" not in up["title"])
check("hostile strings: an over-long acceptance is ignored, the finding stays open", [f for f in findings(eng, POSTGRES) if f["id"] == "CVE-2026-0010"][0]["status"] != "accepted")
check("hostile strings: an over-long acceptance degrades the run", eng.report["run"]["status"] == "degraded")
check("hostile strings: the degraded run is not recorded as complete", eng.state["lastCompleteRunAt"] is None)
with tempfile.TemporaryDirectory() as tmp:
# Isolation: values already in state, whatever put them there, cannot refuse the whole report.
eng, runner, clock = engine(tmp)
eng.tick()
clean = report.build(eng.state, [], [], clock())
check("mutation: a clean state is not reported as repaired", clean["run"]["status"] == "complete" and clean == report.build(eng.state, [], [], clock()))
record = eng.state["scans"][A1]
record["findings"][0].update(url="HTTP://x", title="t" * 900, severity="weird", cvss=99, package="p" * 900)
eng.state["database"] = {"updatedAt": "not-a-time", "version": "1"}
eng.state["remediation"] = [{"image": "a/b", "state": "pr-open", "url": "HTTP://x", "updatedAt": "2026-11-09T00:00:00Z"}]
target = next(i for i in eng.state["inventory"] if i["id"] == NIGHTLY)
target["id"] = "ghcr.io/" + "o" * 3000 + ":1"
target["image"] = "ghcr.io/" + "o" * 3000
eng.state["scans"][target["id"]] = eng.state["scans"].pop(NIGHTLY) # a by-tag unit is keyed by its id
built = report.build(eng.state, [], [], clock())
check("isolation: the repaired report is schema-valid", report.validate(built) == [])
check("isolation: a repair degrades the run", built["run"]["status"] == "degraded")
bad = [f for f in built["findings"] if f["targetId"] == NGINX and f["title"] == "t" * 512][0]
check("isolation: the bad finding survives with sanitised fields",
bad["url"] == "" and bad["severity"] == "UNKNOWN" and bad["cvss"] is None and len(bad["package"]) == 256)
check("isolation: every other finding is untouched", len(built["findings"]) == len(clean["findings"]))
check("isolation: a bad remediation loses only its url", built["remediation"][0]["url"] is None)
check("isolation: an unusable database is reported as unavailable", built["database"]["version"] == "unavailable")
long_ids = [t for t in built["targets"] if len(t["id"]) > 2000]
check("isolation: an over-long target stays listed", len(long_ids) == 1 and len(long_ids[0]["id"]) <= 2048)
check("isolation: findings follow the shortened target id",
any(f["targetId"] == long_ids[0]["id"] for f in built["findings"]))
class FakeProcess:
returncode = 0
def __init__(self, stdout):
self.stdout = stdout
class VersionRunner(trivy.Runner):
def __init__(self, stdout):
super().__init__("/nonexistent", "/nonexistent")
self.stdout = stdout
def _run(self, args, timeout):
return FakeProcess(self.stdout)
def db_info(updated, version):
return VersionRunner(json.dumps({"VulnerabilityDB": {"UpdatedAt": updated, "Version": version}})).db_info()
check("db_info rejects an unparseable timestamp", db_info("yesterday", "2") is None)
check("db_info rejects a non-string timestamp", db_info(5, "2") is None)
check("db_info bounds the version", len(db_info("2026-11-09T06:13:52.123456789Z", "v" * 500)["version"]) == 128)
check("mutation: db_info keeps a good database", db_info("2026-11-09T06:13:52Z", "2") == {"updatedAt": "2026-11-09T06:13:52Z", "version": "2"})
check("a credential failure's reason is bounded", len(trivy.classify_failure("r" * 3000, "401 unauthorized").reason) <= 1024)
for url, want in (("https://a.test/x", "https://a.test/x"), ("http://a.test/", "http://a.test/"), ("HTTPS://a.test/x", ""),
("HTTP://a.test/x", ""), ("Https://a.test/x", ""), ("javascript:alert(1)", ""), ("", "")):
check("safe_url " + repr(url), trivy.safe_url(url) == want)
check("every url safe_url keeps satisfies the report schema",
report.re.search(report.SCHEMA["properties"]["findings"]["items"]["properties"]["url"]["pattern"], trivy.safe_url("https://a.test/x")))
# Recorded Trivy output through the normaliser: untrusted text.
parsed = trivy.parse_results({"SchemaVersion": 2, "Results": [{"Vulnerabilities": [
{"VulnerabilityID": "CVE-1", "Severity": "weird", "PkgName": "p", "InstalledVersion": "1", "Title": "x\x07y",
+46 -13
View File
@@ -17,17 +17,48 @@ Hand-authored platform tooling (not a catalog service), reconciled by its own no
The page is `cve.<domain>`, internal only, gated by the per-pod oauth2-proxy sidecar to `platform-admins`.
The route is disowned from public DNS and the tunnel 404s it; PiHole answers it on the LAN.
## Operator step: add the Keycloak client (NOT done by the change that added this tree)
## Operator action required: the Keycloak client (R-VULN-04 is NOT delivered until this is done)
This change adds everything around the Keycloak client but not the client. Adding it needs a new key in
`platform/keycloak/keycloak-realm-clients.sops.yaml`, an encrypted file an agent must not decrypt; the
L0 check `keycloak-client-secret-reconciler` then requires the client and that key to appear in the
same change. Until this is done the page cannot complete a login (`invalid_client`, fail-closed); the
Until these steps are done the page cannot complete a login (`invalid_client`, fail-closed). The
scanner, its metrics and its alerts are unaffected.
In one PR (the realm import does not re-run on its own for a NEW client):
An agent cannot do this, because every route runs through one encrypted file. The client's secret
must be the key `vuln-scanner-client-secret` in `platform/keycloak/keycloak-realm-clients.sops.yaml`:
the reconciler mounts only that Secret, and the L0 check `keycloak-client-secret-reconciler` rejects a
placeholder that reads any other (a second source would be declared but never healed). That check
also requires the client and the key to land in the same change. Adding a key needs the age private
key. Separately, a completed realm import never re-runs, so a new client has to be created in the
live realm once whatever git says. `*.sops.yaml` is a sensitive path, so the PR needs an approval or
an operator-posted `/approve-override` (`infra/gitea/approval-policy.json`).
1. Generate one value and set it in both encrypted files, through pipes so it is never echoed:
**Order matters.** Create the live client first: the reconciler treats every `*-client-secret` key as
a client that must exist, and fails every 15-minute run with `NOT FOUND` otherwise
(`KeycloakClientSecretReconcilerFailing`).
1. Create the client in the live realm. No secret is passed: the reconciler sets it from git after
the merge, and it reads the admin credential from the pod's own environment, so nothing is echoed.
```bash
kubectl --context k3s-prod -n keycloak exec keycloak-0 -- bash -c '
K="/opt/keycloak/bin/kcadm.sh"; C="--config /tmp/kcadm-vuln.config"
$K config credentials $C --server http://localhost:8080 --realm master \
--user "$KC_BOOTSTRAP_ADMIN_USERNAME" --password "$KC_BOOTSTRAP_ADMIN_PASSWORD"
D=home.keithjackson.dev
ID=$($K create clients $C -r homelab -i -s clientId=vuln-scanner -s "name=CVE report (oauth2-proxy)" \
-s protocol=openid-connect -s enabled=true -s publicClient=false -s standardFlowEnabled=true \
-s directAccessGrantsEnabled=false -s "redirectUris=[\"https://cve.$D/oauth2/callback\"]" \
-s "webOrigins=[\"https://cve.$D\"]")
$K create clients/$ID/protocol-mappers/models $C -r homelab -s name=groups -s protocol=openid-connect \
-s protocolMapper=oidc-group-membership-mapper -s "config.\"full.path\"=false" \
-s "config.\"id.token.claim\"=true" -s "config.\"access.token.claim\"=true" \
-s "config.\"userinfo.token.claim\"=true" -s "config.\"claim.name\"=groups"
rm -f /tmp/kcadm-vuln.config'
```
`D` is the prod `BASE_DOMAIN`; check it against the `cluster-vars` ConfigMap first.
2. On a branch, generate one value and set it in both encrypted files, through pipes so it is never
echoed. `client-secret` in the scanner's file is replaced so that the two match:
```bash
./infra/devcontainer/dev.sh run bash -c '
@@ -38,7 +69,7 @@ In one PR (the realm import does not re-run on its own for a NEW client):
"[\"stringData\"][\"client-secret\"]" "\"$cs\""'
```
2. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client:
3. Add to `platform/keycloak/keycloak-realm-import.yaml`, beside the `crafty` client:
```yaml
# spec.placeholders
@@ -72,9 +103,11 @@ In one PR (the realm import does not re-run on its own for a NEW client):
"claim.name": groups
```
3. Regenerate the derived URL file: `python3 ci/render-keycloak-client-urls.py --write`. Commit all
three, gate, merge.
4. Regenerate the derived URL file with `python3 ci/render-keycloak-client-urls.py --write`, and add
`vuln-scanner-client-secret: REPLACE-ME` to `keycloak-realm-clients.sops.yaml.example`. Commit all of
it in one PR, gate, merge.
4. After merge, the client must also exist in the live realm (a completed realm import does not re-run,
per `platform/keycloak/README.md`, so create it once with `kcadm` or re-trigger the import). The
client-secret reconciler then converges its secret and URLs.
5. After Flux applies it, the next reconciler run sets the client's secret and URLs from git. Prove it
with the L2 test `keycloak-client-secrets-match-git`, then log in at `https://cve.<domain>`. The
`platform-admins` restriction is the sidecar's `--allowed-group`; the `groups` mapper above is what
supplies the claim it reads, so a user outside that group gets a 403 from oauth2-proxy.
+7 -1
View File
@@ -133,6 +133,12 @@ spec:
value: /tools/ca-certificates.crt
- name: VULN_ACCEPTANCES
value: /config/acceptances.yaml
# Registries that need a credential this scanner deliberately does not hold (R-VULN-09).
# Comma-separated hostnames. They report credential-required WITHOUT invoking Trivy:
# the egress policy excludes the LAN, so a real attempt ends in a connection error and
# would read as `failed`, firing VulnCoverageGap permanently. Flux substitutes the domain.
- name: VULN_PRIVATE_REGISTRIES
value: gitea.${BASE_DOMAIN}
- name: PYTHONUNBUFFERED
value: "1"
- name: PYTHONDONTWRITEBYTECODE
@@ -222,7 +228,7 @@ spec:
env:
# client-secret MUST equal the Keycloak `vuln-scanner` client's secret (key
# vuln-scanner-client-secret in platform/keycloak/keycloak-realm-clients.sops.yaml; see
# README.md, Operator step). Secrets by name only (R-IDP-04).
# README.md, Operator action required). Secrets by name only (R-IDP-04).
- name: OAUTH2_PROXY_CLIENT_ID
valueFrom:
secretKeyRef: { name: vuln-scanner-oauth2-proxy, key: client-id }
+3
View File
@@ -15,6 +15,7 @@ from datetime import date, timedelta
from state import utcnow
MAX_DAYS = 180
MAX_REASON = 2048 # report.schema.json's limit; a longer reason would make the whole report invalid
_KEYS = ("id", "image", "reason", "expires")
_ENTRY = re.compile(r"^- ([a-z]+):(?: (.*))?$")
_FIELD = re.compile(r"^ ([a-z]+):(?: (.*))?$")
@@ -91,6 +92,8 @@ def _validate(fields, today):
problems.append("id is missing or malformed")
if not fields.get("reason"):
problems.append("reason is mandatory")
elif len(fields["reason"]) > MAX_REASON:
problems.append("reason is longer than " + str(MAX_REASON) + " characters")
if "image" in fields and not fields["image"]:
problems.append("image is empty")
expires = fields.get("expires") or ""
+62 -5
View File
@@ -23,15 +23,21 @@ log = logging.getLogger("vuln-scanner")
FULL_RUN_INTERVAL = timedelta(hours=20)
DB_MAX_AGE = timedelta(hours=48)
DB_REFRESH_AGE = timedelta(hours=24)
KILLED_REASON = "scanner killed during scan (OOM/eviction?)"
class Engine:
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120):
def __init__(self, data_dir, client, runner, acceptances_path, clock=st.utcnow, tick_seconds=900, publish_seconds=120,
private_registries=()):
self.store = st.Store(data_dir)
self.data_dir = data_dir
self.client = client
self.runner = runner
self.acceptances_path = acceptances_path
# Registries known to need credentials the scanner deliberately does not hold (R-VULN-09) and,
# being on the LAN, not reachable from its egress policy: a scan would end in a connection
# error (`failed`) rather than a 401 (`credential-required`), so it is never attempted.
self.private_registries = {name.strip().lower() for name in private_registries if name.strip()}
self.clock = clock
self.tick_seconds = tick_seconds
self.publish_seconds = publish_seconds
@@ -43,6 +49,18 @@ class Engine:
self.loop_thread = None
self._last_publish = 0.0
self._last_inventory = None
self._recover_interrupted_scan()
def _recover_interrupted_scan(self):
"""An inFlight key at startup means the last process died inside that scan."""
key = self.state.get("inFlight")
if not key:
return
log.error("the previous process was killed while scanning %s; marking it failed", key)
record = self.state["scans"].get(key)
if record is not None:
record["killedAt"] = fmt_ts(self.clock())
self._record_failure(key, "failed", KILLED_REASON)
# --- views used by the HTTP servers; none of them waits for a scan --------------------------
def report_bytes(self):
@@ -67,16 +85,19 @@ class Engine:
# --- persistence -----------------------------------------------------------------------------
def publish(self):
"""True when the report was written; False when it was refused and the previous one stands."""
with self.lock:
now = self.clock()
entries, errors = acceptances.load(self.acceptances_path, now.date())
for problem in errors:
log.error("invalid acceptance ignored: %s", problem)
built = report_mod.build(self.state, entries, errors, now)
if report_mod.publish(built, self.data_dir):
published = report_mod.publish(built, self.data_dir)
if published:
self.report = built
self.report_json = report_mod.serialise(built)
self._last_publish = time.monotonic()
return published
# --- inventory -------------------------------------------------------------------------------
def refresh_inventory(self):
@@ -152,6 +173,14 @@ class Engine:
return timedelta.max if moment is None else self.clock() - moment
def _scan_unit(self, key, reference):
registry = trivy.registry_of(reference)
if registry in self.private_registries or registry.split(":", 1)[0] in self.private_registries:
self._record_failure(key, "credential-required",
"registry " + registry + " is private and the scanner holds no credential for it")
return
with self.lock:
self.state["inFlight"] = key
self.store.save(self.state)
try:
found = trivy.parse_results(self.runner.scan(reference))
except trivy.ScanFailure as exc:
@@ -166,11 +195,24 @@ class Engine:
except Exception: # pylint: disable=broad-except
log.exception("scan cache cleanup failed")
def _finish_in_flight(self, key):
"""Cleared in the same write as the scan's outcome, so no crash can separate the two."""
if self.state.get("inFlight") == key:
self.state["inFlight"] = None
def _killed_recently(self, key):
"""A unit that killed the scanner sits out full runs for a day: retrying it first on every
restart would OOM again before any later image, which is the wedge this exists to prevent."""
killed = parse_ts(self.state["scans"][key].get("killedAt"))
return killed is not None and self.clock() - killed < FULL_RUN_INTERVAL
def _record_failure(self, key, kind, reason):
now = fmt_ts(self.clock())
with self.lock:
self._finish_in_flight(key)
record = self.state["scans"].get(key)
if record is None:
self.store.save(self.state)
return
record["lastAttemptAt"] = now
record["outcome"] = kind
@@ -181,10 +223,13 @@ class Engine:
def _record_success(self, key, found):
now = fmt_ts(self.clock())
with self.lock:
self._finish_in_flight(key)
record = self.state["scans"].get(key)
if record is None:
self.store.save(self.state)
return
record["lastAttemptAt"] = now
record["killedAt"] = None
record["lastSuccessAt"] = now
record["outcome"] = "ok"
record["reason"] = None
@@ -215,7 +260,10 @@ class Engine:
started = self.clock()
log.info("full run starting")
database_ok = self._refresh_database(always=True)
self._scan_queue(sorted(self._units()))
self._scan_queue([key for key in sorted(self._units()) if not self._killed_recently(key)])
if self.stop.is_set():
log.info("full run interrupted by shutdown; not recorded")
return
info = self.state["database"]
now = self.clock()
degraded = not database_ok or info is None or self._db_age(info) > DB_MAX_AGE
@@ -225,8 +273,15 @@ class Engine:
status = "degraded" if degraded else "complete"
with self.lock:
self.state["run"] = {"startedAt": fmt_ts(started), "finishedAt": fmt_ts(now), "status": status}
if not degraded:
self.store.save(self.state)
# Only a report that actually reached the store counts as a completed run. Otherwise a
# refused report would freeze the page while VulnScanStale stayed quiet.
if self.publish() and not degraded and self.report["run"]["status"] == "complete":
self.state["lastCompleteRunAt"] = fmt_ts(now)
# A corrupt state.json degrades the run that rebuilds from it, and no further: once a full
# run has completed against a good inventory, the state is whole again.
if self.state.get("loadError") and self._last_inventory is not None and not self.state.get("inventoryError"):
self.state["loadError"] = None
self.store.save(self.state)
log.info("full run finished: %s", status)
@@ -262,7 +317,9 @@ def main():
tools_dir = os.environ.get("VULN_TOOLS_DIR", "/tools")
trivy_bin = os.environ.get("TRIVY_BIN", os.path.join(tools_dir, "trivy"))
runner = trivy.Runner(trivy_bin, cache_dir, int(os.environ.get("VULN_SCAN_TIMEOUT_SECONDS", "600")))
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"))
private_registries = os.environ.get("VULN_PRIVATE_REGISTRIES", "").split(",")
engine = Engine(data_dir, inventory.KubeClient(), runner, os.environ.get("VULN_ACCEPTANCES", "/config/acceptances.yaml"),
private_registries=private_registries)
server.start_all(engine, ("127.0.0.1", 8080), ("0.0.0.0", 8081), ("0.0.0.0", 9090))
finished = threading.Event()
signal.signal(signal.SIGTERM, lambda *_: finished.set())
+35 -12
View File
@@ -180,14 +180,39 @@ def _sev(severity):
return '<span class="sev-' + e(severity) + '">' + e(severity) + "</span>"
def _workload_lines(target):
def _state_label(target):
return "target " + e(target["state"]) + (" (" + e(target["reason"]) + ")" if target.get("reason") else "")
def _status_label(finding):
"""One finding's own status, with its own acceptance: never another image's."""
label = "status " + e(finding["status"])
acceptance = finding.get("acceptance")
if acceptance:
label += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
return label
def _workload_lines(target, finding=None):
"""One line per workload, each carrying the target state and, in a CVE expansion, the status."""
suffix = " — " + _state_label(target) + (" — " + _status_label(finding) if finding else "")
items = []
for workload in target["workloads"]:
items.append("<li>" + e(workload["namespace"]) + " / " + e(workload["kind"]) + " / " + e(workload["name"])
+ " — containers: " + e(", ".join(workload["containers"]) or "none") + "</li>")
+ " — containers: " + e(", ".join(workload["containers"]) or "none") + suffix + "</li>")
return "<ul>" + "".join(items) + "</ul>"
def _status_summary(statuses):
"""The row-level status. A CVE accepted on one image and open on another reads as mixed."""
counts = {}
for status in statuses:
counts[status] = counts.get(status, 0) + 1
if len(counts) == 1:
return e(next(iter(counts)))
return "mixed: " + ", ".join(e(counts[s]) + " " + e(s) for s in sorted(counts, key=STATUS_RANK.get))
def _remediation_cell(images, remediation):
if not images:
return ""
@@ -250,13 +275,10 @@ def _cves(report, targets, params, remediation):
for finding, target in row["items"]:
if finding["status"] == "fix-available":
fix_images.add(target["image"])
lines.append("<li>" + e(target["id"]) + " [" + e(target["state"]) + "] — " + e(finding["package"]) + " "
+ e(finding["installedVersion"]) + " → " + e(finding["fixedVersion"] or "no fix")
+ _workload_lines(target) + "</li>")
acceptance = next((f.get("acceptance") for f, _ in row["items"] if f.get("acceptance")), None)
status_text = e(row["status"])
if acceptance:
status_text += " (" + e(acceptance["reason"]) + ", until " + e(acceptance["expires"]) + ")"
lines.append("<li>" + e(target["id"]) + " — " + _state_label(target) + " — " + _status_label(finding)
+ " — " + e(finding["package"]) + " " + e(finding["installedVersion"]) + " → "
+ e(finding["fixedVersion"] or "no fix") + _workload_lines(target, finding) + "</li>")
status_text = _status_summary(f["status"] for f, _ in row["items"])
html_rows.append(
"<tr><td>" + _link(row["url"], row["id"]) + "</td><td>" + _sev(row["severity"]) + "</td><td>"
+ e("" if row["cvss"] is None else row["cvss"]) + "</td><td>" + e(row["package"]) + "</td><td>" + e(row["fix"])
@@ -317,12 +339,13 @@ def _workloads(report, targets, params):
html_rows = []
for row in page_rows:
lines = "".join(
"<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + e(f["status"]) + "</li>"
for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"])))
"<li>" + _link(f["url"], f["id"]) + " " + _sev(f["severity"]) + " " + e(f["package"]) + " — " + _status_label(f)
+ " — " + e(f["targetId"]) + " — " + _state_label(targets[f["targetId"]]) + "</li>"
for f in sorted(row["findings"].values(), key=lambda f: (SEV_RANK[f["severity"]], f["id"], f["targetId"])))
counts = "".join("<td>" + e(row["counts"][s]) + "</td>" for s in SEVERITIES)
html_rows.append(
"<tr><td>" + e(row["key"][0]) + "</td><td>" + e(row["key"][1]) + "</td><td>" + e(row["key"][2]) + "</td><td>"
+ e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) for i in sorted(row["images"])) + "</td>"
+ e(", ".join(sorted(row["containers"])) or "none") + "</td><td>" + "<br>".join(e(i) + " — " + _state_label(targets[i]) for i in sorted(row["images"])) + "</td>"
+ counts + "<td><details><summary>" + e(len(row["findings"])) + " findings</summary><ul>" + lines + "</ul></details></td></tr>")
head = [_th(params, "Namespace", "namespace"), _th(params, "Kind", "kind"), _th(params, "Workload", "name"), "<th>Containers</th>",
_th(params, "Images", "images")] + [_th(params, s, s.lower()) for s in SEVERITIES] + [_th(params, "Findings", "total")]
+89 -1
View File
@@ -1,6 +1,7 @@
"""Build, validate and publish report.json, the one interface to the page and the bot."""
import json
import logging
import hashlib
import os
import re
from datetime import date, timedelta
@@ -85,10 +86,15 @@ def build(state, accepted, acceptance_errors, now):
if acceptance_errors or state.get("inventoryError") or state.get("loadError") \
or db_time is None or now - db_time > DB_STALE:
run["status"] = "degraded"
return {
built = {
"schemaVersion": 1, "generatedAt": fmt_ts(now), "database": database, "run": run,
"targets": targets, "findings": findings, "remediation": list(state["remediation"]),
}
repaired = _isolate(built)
if repaired:
log.error("%d report entries held a value the schema refuses; repaired or dropped them", repaired)
run["status"] = "degraded"
return built
# --- hand validator for the JSON-Schema subset report.schema.json uses -----------------------------
@@ -175,6 +181,88 @@ def _check(schema, value, path, errors):
_check(branch, value, path, errors)
# --- isolation: one bad upstream value costs that value, never the whole report ---------------------
def _repair(schema, value):
"""(usable, value): value itself when valid, else the nearest valid form of it."""
trial = []
_check(schema, value, "$", trial)
if not trial:
return True, value
if isinstance(value, dict) and "properties" in schema:
fixed = {}
for key, sub in schema["properties"].items():
if key not in value:
continue
usable, item = _repair(sub, value[key])
if usable:
fixed[key] = item
elif key in schema.get("required", []):
return False, None
trial = []
_check(schema, fixed, "$", trial)
return not trial, fixed
if isinstance(value, list) and "items" in schema:
repaired = [_repair(schema["items"], item) for item in value]
return True, [item for usable, item in repaired if usable]
candidates = [None, "", "UNKNOWN"]
if isinstance(value, str) and "maxLength" in schema:
candidates.insert(0, value[:schema["maxLength"]])
for candidate in candidates:
trial = []
_check(schema, candidate, "$", trial)
if not trial:
return True, candidate
return False, None
def _isolate(report):
"""Repair (or, as a last resort, drop) each entry the schema refuses. Returns how many changed."""
changed = 0
schema = SCHEMA["properties"]
usable, database = _repair(schema["database"], report["database"])
if database is not report["database"]:
changed += 1
report["database"] = database if usable else {"updatedAt": EPOCH, "version": "unavailable"}
renamed, dropped, targets = {}, set(), []
for target in report["targets"]:
original = target["id"]
if len(original) > 2048: # keep it unique: a truncated id may not collide with another
target = dict(target, id=original[:2031] + "#" + hashlib.sha256(original.encode("utf-8")).hexdigest()[:16])
renamed[original] = target["id"]
usable, fixed = _repair(schema["targets"]["items"], target)
if not usable:
changed += 1
dropped.add(original)
continue
if fixed is not target or original in renamed:
changed += 1
targets.append(fixed)
report["targets"] = targets
findings = []
for finding in report["findings"]:
if finding["targetId"] in dropped:
changed += 1
continue
if finding["targetId"] in renamed:
finding = dict(finding, targetId=renamed[finding["targetId"]])
usable, fixed = _repair(schema["findings"]["items"], finding)
if not usable or fixed is not finding:
changed += 1
if usable:
findings.append(fixed)
report["findings"] = findings
remediation = []
for entry in report["remediation"]:
usable, fixed = _repair(schema["remediation"]["items"], entry)
if not usable or fixed is not entry:
changed += 1
if usable:
remediation.append(fixed)
report["remediation"] = remediation
return changed
def validate(report):
"""Schema and referential errors; an empty list means the report may be served."""
errors = []
+3 -1
View File
@@ -126,7 +126,9 @@ def validate_triage(document):
if entry["state"] not in REMEDIATION_STATES:
return [prefix + "unknown state"]
url = entry["url"]
if url is not None and (not isinstance(url, str) or len(url) > 2048 or trivy.safe_url(url) != url):
# safe_url is the same rule the report schema enforces, so a value accepted here cannot make
# the report invalid. Empty is refused: safe_url returns "" for it, and the schema does not.
if url is not None and (not isinstance(url, str) or not url or trivy.safe_url(url) != url):
return [prefix + "url must be null or an http(s) URL"]
if parse_ts(entry["updatedAt"]) is None or len(entry["updatedAt"]) > 64:
return [prefix + "updatedAt must be RFC 3339"]
+4
View File
@@ -72,6 +72,9 @@ def new_state():
"remediation": [],
"triageHeartbeatAt": None,
"loadError": None,
# The scan key Trivy is running against right now, persisted BEFORE the scan starts: an OOM
# kill takes the whole container, so nothing can record the failure afterwards.
"inFlight": None,
}
@@ -84,6 +87,7 @@ def new_record(now):
"outcome": None,
"reason": None,
"failedSince": None,
"killedAt": None,
"findings": [],
"firstSeen": {},
}
+19 -8
View File
@@ -5,6 +5,8 @@ import re
import subprocess
from urllib.parse import urlsplit
from state import parse_ts
log = logging.getLogger("vuln-scanner.trivy")
SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
@@ -13,7 +15,11 @@ _CREDENTIAL = re.compile(
r"|status(?: code)?:? ?(?:401|403)\b|\b40[13] (?:unauthorized|forbidden)",
re.IGNORECASE,
)
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
# Lone surrogates too: json.loads accepts them, and they cannot be encoded as UTF-8 for the page.
_CONTROL = re.compile(r"[\x00-\x1f\x7f\ud800-\udfff]")
# Exact and lowercase, matching report.schema.json, whose pattern is case-sensitive: an `HTTPS://x`
# accepted here would make the schema refuse the entire report.
_URL_SCHEME = re.compile(r"https?://")
class ScanFailure(Exception):
@@ -35,13 +41,18 @@ def text(value, limit=512):
def safe_url(value):
"""value if it is a plain http(s) URL, else the empty string."""
url = text(value, 2048)
if not url or url != value or re.search(r"\s", url):
if not url or url != value or re.search(r"\s", url) or not _URL_SCHEME.match(url):
return ""
try:
parts = urlsplit(url)
except ValueError:
return ""
return url if parts.scheme.lower() in ("http", "https") and parts.netloc else ""
return url if parts.netloc else ""
def registry_of(reference):
"""The lowercase registry host (and port) of a normalised registry/repository reference."""
return reference.split("/", 1)[0].lower()
def _cvss(vuln):
@@ -95,7 +106,7 @@ def parse_results(document):
def classify_failure(registry, stderr):
"""Sort a non-zero Trivy exit into credential-required or failed. Reasons name the registry."""
if _CREDENTIAL.search(stderr or ""):
return ScanFailure("credential-required", "registry " + registry + " refused anonymous access")
return ScanFailure("credential-required", "registry " + text(registry, 253) + " refused anonymous access")
lines = [line.strip() for line in (stderr or "").splitlines() if line.strip()]
return ScanFailure("failed", text(lines[-1], 400) if lines else "trivy exited with an error and no message")
@@ -125,9 +136,9 @@ class Runner:
db = json.loads(result.stdout).get("VulnerabilityDB")
except (ValueError, AttributeError):
return None
if not isinstance(db, dict) or not db.get("UpdatedAt"):
return None
return {"updatedAt": db["UpdatedAt"], "version": str(db.get("Version", "unknown"))}
if not isinstance(db, dict) or parse_ts(db.get("UpdatedAt")) is None:
return None # an unparseable timestamp is no usable database, and must not reach the report
return {"updatedAt": db["UpdatedAt"], "version": text(str(db.get("Version", "unknown")), 128) or "unknown"}
def scan(self, reference):
args = [
@@ -143,7 +154,7 @@ class Runner:
if result.returncode < 0:
raise ScanFailure("failed", "trivy killed by signal " + str(-result.returncode) + " (out of memory?)")
if result.returncode != 0:
raise classify_failure(reference.split("/", 1)[0], result.stderr)
raise classify_failure(registry_of(reference), result.stderr)
try:
return json.loads(result.stdout)
except ValueError:
+10
View File
@@ -483,6 +483,16 @@
"description": "ADDED 2026-10-30. MAJORS WAIT FOR A TICK ON THE DEPENDENCY DASHBOARD instead of opening a PR. They are never auto-merged anyway, and with prConcurrentLimit 5 the dry run's 10 majors would hold every PR slot open indefinitely, starving the patch/minor/digest PRs that can merge on green.",
"matchUpdateTypes": ["major"],
"dependencyDashboardApproval": true
},
// ADDED 2026-11-09 — EVERY IMAGE IN THE SCANNER POD, NOT ONLY TRIVY. The python image runs the
// process that holds the cluster-read token (rbac.yaml, R-VULN-09), and without this rule its
// digest and patch bumps fall through to the auto-merge rule above. LAST ON PURPOSE: Renovate
// applies packageRules in order and a later match overrides, so nothing above can undo it.
{
"description": "Anything under platform/vuln-scanner: the python image holds the cluster-read token. Never auto-merged, 7-day minimum release age (R-VULN-09).",
"matchFileNames": ["platform/vuln-scanner/**"],
"automerge": false,
"minimumReleaseAge": "7 days"
}
],
// WHAT RENOVATE STILL CANNOT FIX, so nobody mistakes a working bot for full coverage. A