# ServiceDefinition: nextcloud # Source of truth: spec docs/13-service-catalog.md row 7; docs/15-identity-and-access.md; docs/04. # public file-sync | egress internet | data on Longhorn (irreplaceable, DR-critical) | auth oidc. # # DR-critical (docs/13 "DR-critical data"): the user data lives on the 3-replica `longhorn` tier and # is backed up off-cluster (backup.policy daily-offcluster; R-STATE-03). Nextcloud is stateless code # over (a) its /var/www/html Longhorn PVC and (b) an external PostgreSQL — the first CNPG-DB app, # establishing the reusable pattern (its DB is a CloudNativePG Cluster, apps/nextcloud-db.yaml). # # DB WIRING (schema >=1.2.0 env/envFrom): the app reaches its PostgreSQL cross-namespace at # nextcloud-db-rw.nextcloud-db:5432 (the CNPG read-write Service). Its owner credentials are the # SAME password held in two SOPS secrets — nextcloud-db-app (basic-auth, ns nextcloud-db; consumed by # the CNPG Cluster's initdb) and nextcloud-bootstrap (ns nextcloud; consumed here as db-password). # A Pod can only read Secrets in its OWN namespace, so the shared password is duplicated across the # two encrypted files rather than replicated cross-namespace (documented in apps/nextcloud-db.yaml). # Validates against schemas/servicedefinition.schema.json. name: nextcloud kind: container source: image: nextcloud:31-apache # apache variant serves on :80 (docs/13 row 7); pinned recent stable # major (R-UPD-01); update.range governs advancement within the line env: # --- PostgreSQL connection (the CNPG-DB pattern). HOST/DB/USER are literals; PASSWORD is sourced # from the in-namespace SOPS secret (the CNPG -app secret lives in ns nextcloud-db, unreachable # by a Pod secretKeyRef, so the same value is mirrored into nextcloud-bootstrap here). --- - name: POSTGRES_HOST value: nextcloud-db-rw.nextcloud-db.svc.cluster.local - name: POSTGRES_DB value: nextcloud - name: POSTGRES_USER value: nextcloud - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: nextcloud-bootstrap key: db-password # --- Break-glass local admin (docs/15 availability caveat): created on first-boot install so the # operator is never locked out by an IdP outage. Sourced from the SOPS nextcloud-bootstrap secret. --- - name: NEXTCLOUD_ADMIN_USER valueFrom: secretKeyRef: name: nextcloud-bootstrap key: admin-username - name: NEXTCLOUD_ADMIN_PASSWORD valueFrom: secretKeyRef: name: nextcloud-bootstrap key: admin-password # --- Public exposure behind the TLS-terminating shared Gateway (docs/08/09). TRUSTED_DOMAINS lets # the vhost answer for the public FQDN; OVERWRITE* make Nextcloud emit https:// self-URLs even # though it speaks plain HTTP behind the proxy. --- - name: NEXTCLOUD_TRUSTED_DOMAINS value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEHOST value: nextcloud.${BASE_DOMAIN} - name: OVERWRITEPROTOCOL value: https - name: OVERWRITECLIURL value: https://nextcloud.${BASE_DOMAIN} # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential # `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret # (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it # via user_oidc's --clientsecret-env (never on argv) to register the "keycloak" provider. The LOCAL # admin above stays as the break-glass path (docs/15). --- - name: NEXTCLOUD_OIDC_CLIENT_SECRET valueFrom: secretKeyRef: name: nextcloud-bootstrap key: oidc-client-secret lifecycle: postStart: exec: command: - /bin/sh - -c - | # Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint # (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR # updates the "keycloak" login provider (occ user_oidc:provider is create-or-update by # identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud # group provisioning. occ runs as www-data (owns config.php); the client secret is passed via # --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing # secret or not-yet-installed instance is skipped so the pod never crashloops (the local # break-glass admin still works). set -u # OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*, # so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The # provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as # ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument. OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; } NEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration nextcloud_oidc_fail() { echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2 { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true } # (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds # maintenance mode during first-boot install/upgrade; occ writes made in that window do not # stick — a second race on top of the quoting bug fixed at (c). i=0 until OCC status 2>/dev/null | grep -q "installed: true" && ! OCC status 2>/dev/null | grep -q "maintenance: true"; do i=$((i + 1)) [ "$i" -ge 300 ] && { echo "[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping" >&2; exit 0; } sleep 2 done # Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak # FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's # SSRF guard otherwise REFUSES that request — OCP\Http\Client\LocalServerException "violates # local access rules" — during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to # the user as "Could not reach the provider". (The (b) curl check below uses system libcurl, # NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied # every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27. OCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \ || echo "[nextcloud-oidc] WARN: failed to set allow_local_remote_servers" >&2 # NOTE the `$$` escape below — it is load-bearing. apps-wave-1 declares postBuild. # substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE # time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is # not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z "" ]` — ALWAYS # TRUE — so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never # installing/enabling user_oidc nor registering the provider. SSO kept working only because # the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with # NO OIDC at all. Flux STRICT substitution cannot catch this — it EXEMPTS default-operator # forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion # for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping — which is why # `--clientsecret-env` and the bare uses below always worked and only the guard broke. # Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py. if [ -z "$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}" ]; then echo "[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)" exit 0 fi if ! OCC app:list 2>/dev/null | grep -q "user_oidc"; then OCC app:install user_oidc || echo "[nextcloud-oidc] app:install user_oidc failed" >&2 fi OCC app:enable user_oidc >/dev/null 2>&1 || true # (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering. d=0; disc_ok=0 while [ "$d" -lt 45 ]; do if [ "$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$NEXTCLOUD_DISCOVERY_URL" 2>/dev/null)" = "200" ]; then disc_ok=1; break fi d=$((d + 1)); sleep 2 done [ "$disc_ok" = 1 ] || nextcloud_oidc_fail "Keycloak discovery endpoint not reachable (HTTP 200) within timeout" # (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle. # HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a # single argument. The previous `OCC user_oidc:provider ... --scope="openid email profile"` # form fed the value back through the wrapper's $*, which SPLIT it into extra positionals -> # occ "Too many arguments" -> silent failure on EVERY boot. Secret via --clientsecret-env # (never on argv). The verify-then-retry loop guarantees the provider actually persisted. r=0; reg_ok=0 while [ "$r" -lt 10 ]; do su -s /bin/sh www-data -c "php /var/www/html/occ user_oidc:provider keycloak \ --clientid=nextcloud \ --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \ --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \ --scope='openid email profile' \ --unique-uid=0 \ --mapping-uid=preferred_username \ --mapping-email=email \ --mapping-display-name=name \ --group-provisioning=1 \ --mapping-groups=groups" >/dev/null 2>&1 || true if OCC user_oidc:provider 2>/dev/null | grep -q "keycloak"; then reg_ok=1; break fi r=$((r + 1)); sleep 3 done if [ "$reg_ok" = 1 ]; then echo "[nextcloud-oidc] provider 'keycloak' configured and verified" else nextcloud_oidc_fail "user_oidc:provider ran but the provider did not persist after retries" fi # (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has # NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a # login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the # built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every # start: occ group:adduser is a no-op for an existing member. A user is promotable only # AFTER their first OIDC login provisions them into the group, so a brand-new admin gains # rights on their next login + the following pod start (or run `occ group:adduser admin # ` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is # passed to php as argv (never interpolated) so group:list JSON parses robustly. ADMIN_SRC_GROUP=platform-admins # HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser` # exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot # transient failure (DB / maintenance race in the first seconds after container start) silently # dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds # any platform-admins member still missing from 'admin' until all are verified (bounded ~30s); # it is a no-op for members already in 'admin'. Group names are passed to php as argv (never # interpolated) so the JSON parses robustly. parse_group() { su -s /bin/sh www-data -c "php /var/www/html/occ group:list --output=json" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,"\n";}}' -- "$1" 2>/dev/null; } p=0 while [ "$p" -lt 15 ]; do current_admins=$(parse_group admin) missing=0 for uid in $(parse_group "$ADMIN_SRC_GROUP"); do if printf '%s\n' "$current_admins" | grep -qxF -- "$uid"; then continue fi missing=1 if OCC group:adduser admin "$uid" >/dev/null 2>&1; then echo "[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)" fi done [ "$missing" = 0 ] && break p=$((p + 1)); sleep 2 done if [ "$p" -ge 15 ]; then nextcloud_oidc_fail "could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries" fi # A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container # (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01: # the final statement used to be `[ "$p" -ge 15 ] && nextcloud_oidc_fail ...`, so on the # SUCCESS path the test returned 1 and became the script's exit status. That was latent for # as long as the OIDC guard above was dead code and exited 0 first; fixing the guard # exposed it immediately. Every failure path here logs loudly and writes a marker file and # deliberately does NOT crash the pod — the break-glass local admin must stay usable. exit 0 exposure: public # edge port-forward + public DNS (docs/09; R-DNS/R-NET) egress: internet # normal WAN egress (docs/03; R-NET-05) — app store, integrity checks storage: - name: data size: 20Gi class: longhorn # ALL Nextcloud state (config/, data/, apps/, custom_apps/) on the mountPath: /var/www/html # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02). # The whole html dir is persisted so config.php + user data survive a # rebuild (the default /data/ would strand them on ephemeral rootfs). # THE LEGACY NEXTCLOUD DATASET (Storage/nextcloud, 270 G) — MOUNTED READ-ONLY, DELIBERATELY, AND # NOT AS THIS INSTANCE'S DATA DIRECTORY. Read this before changing either decision. # # WHAT IS ACTUALLY IN THERE (measured 2026-09-07, not assumed): # /mnt/Storage/nextcloud/nextcloud/config 103 M # /mnt/Storage/nextcloud/nextcloud/config_old 3.4 G # /mnt/Storage/nextcloud/nextcloud/config_backup 267 G <- 99% of the dataset # There is NO `data/` directory. This is the OLD, non-k3s Nextcloud's configuration plus a backup # blob — an ARCHIVE, not a live user-data tree. # # WHY READ-ONLY. Two independent reasons, either sufficient: # 1. The old instance is STILL LIVE on it. The export is currently shared rw to 192.168.1.6 and # 192.168.1.5, and the operator has said the existing non-k3s prod must remain active. Two # Nextcloud instances writing one instance's config directory, each with its own database and # therefore its own oc_filecache, is a corruption scenario with no clean recovery. # 2. Nothing here needs writing. An archive is read. # "Do not grant more than needed" therefore lands on `ro`. If the operator's intent is in fact a # MIGRATION (read this data in, then decommission the old instance), that is a different and # larger change than a mount, and it should be a deliberate one — flip `readOnly` and update the # export at that point, not before. Enforced by the PV's nfs.readOnly, so it is a real `ro` mount. # # WHY NOT /var/www/html/data. That path is the LONGHORN PVC above and is this instance's own # state; mounting a foreign tree into it would have Nextcloud scan and index an archive it does # not own. A neutral path keeps the two completely separate. To expose it to users, add it as a # Nextcloud External Storage (Local) mount pointing at this path — an in-app, per-user decision. # # subPath skips the redundant single-child directory: the dataset root holds exactly one entry, # `nextcloud/`, so mounting that subtree puts config/ config_old/ config_backup/ at the top. # # UID: nothing to do. The tree is 0755 owned 1000:3000 and the mount is read-only, so www-data # (uid 33) can read and traverse it. (fsGroup would be a no-op on an NFS volume in any case.) - name: legacy-archive class: bulk export: /mnt/Storage/nextcloud # the ZFS dataset Storage/nextcloud, on ${BULK_NFS_SERVER} subPath: nextcloud mountPath: /mnt/legacy-nextcloud readOnly: true size: 270Gi # ADVISORY ONLY on a static NFS PV — the dataset's used size. dependencies: - to: nextcloud-db # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02). port: 5432 # Renders the egress half; the DB namespace mirrors keycloak-db (no protocol: tcp # default-deny), so Cilium's default-allow accepts the reciprocal ingress. ingressRoutes: - host: nextcloud.${BASE_DOMAIN} port: 80 # apache HTTP; wildcard TLS terminates at the shared Gateway (docs/08/09) auth: mode: oidc # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02) groups: - platform-admins # access scoped to an authorized group (R-IDP-06) # WIRED (native OIDC): the Keycloak `nextcloud` confidential client (platform/keycloak/keycloak-realm- # import.yaml + live kcadm) is registered as a user_oidc login provider ("keycloak") by the postStart # hook (occ user_oidc:provider), using the client secret mirrored into nextcloud-bootstrap # (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The `groups` claim drives # user_oidc group provisioning (R-IDP-06). Members of `platform-admins` are ELEVATED to Nextcloud # admins by the postStart hook step (d) — user_oidc 8.11 has no native group->admin mapping, so the # provisioned membership is mirrored into the built-in `admin` group idempotently on every start; a # user is promoted after their first login provisions them + the next pod start. Nextcloud KEEPS its # LOCAL admin as the break-glass path so an IdP outage never locks the operator out (docs/15 # "Availability caveat"). Reachability to the IdP at login requires allow_local_remote_servers=true # (set by the hook): the Keycloak FQDN resolves to the private gateway VIP, which Nextcloud's SSRF # guard blocks by default ("Could not reach the provider"). No secret is embedded here. update: channel: stable # R-UPD-01 range: ">=31.0.0 <32.0.0" # stay on the 31.x stable line cadence: "0 4 * * *" autoRollbackOnFailedHealth: true # R-UPD-03 observability: metrics: true slis: - up # file-sync down is a critical alert (docs/10; R-OBS-01) backup: policy: daily-offcluster # DR-critical: data backed up off-cluster (docs/13, docs/04; R-STATE-03)