home-network-k3s-impl/catalog/nextcloud.yaml

# ServiceDefinition: nextcloud
# Source of truth: spec docs/13-service-catalog.md row 7; docs/15-identity-and-access.md; docs/04.
# public file-sync | egress internet | data on Longhorn (irreplaceable, DR-critical) | auth oidc.
#
# DR-critical (docs/13 "DR-critical data"): the user data lives on the 3-replica `longhorn` tier and
# is backed up off-cluster (backup.policy daily-offcluster; R-STATE-03). Nextcloud is stateless code
# over (a) its /var/www/html Longhorn PVC and (b) an external PostgreSQL — the first CNPG-DB app,
# establishing the reusable pattern (its DB is a CloudNativePG Cluster, apps/nextcloud-db.yaml).
#
# DB WIRING (schema >=1.2.0 env/envFrom): the app reaches its PostgreSQL cross-namespace at
# nextcloud-db-rw.nextcloud-db:5432 (the CNPG read-write Service). Its owner credentials are the
# SAME password held in two SOPS secrets — nextcloud-db-app (basic-auth, ns nextcloud-db; consumed by
# the CNPG Cluster's initdb) and nextcloud-bootstrap (ns nextcloud; consumed here as db-password).
# A Pod can only read Secrets in its OWN namespace, so the shared password is duplicated across the
# two encrypted files rather than replicated cross-namespace (documented in apps/nextcloud-db.yaml).
# Validates against schemas/servicedefinition.schema.json.
name: nextcloud
kind: container
source:
  image: nextcloud:31-apache    # apache variant serves on :80 (docs/13 row 7); pinned recent stable
                                # major (R-UPD-01); update.range governs advancement within the line
env:
  # --- PostgreSQL connection (the CNPG-DB pattern). HOST/DB/USER are literals; PASSWORD is sourced
  #     from the in-namespace SOPS secret (the CNPG -app secret lives in ns nextcloud-db, unreachable
  #     by a Pod secretKeyRef, so the same value is mirrored into nextcloud-bootstrap here). ---
  - name: POSTGRES_HOST
    value: nextcloud-db-rw.nextcloud-db.svc.cluster.local
  - name: POSTGRES_DB
    value: nextcloud
  - name: POSTGRES_USER
    value: nextcloud
  - name: POSTGRES_PASSWORD
    valueFrom:
      secretKeyRef:
        name: nextcloud-bootstrap
        key: db-password
  # --- Break-glass local admin (docs/15 availability caveat): created on first-boot install so the
  #     operator is never locked out by an IdP outage. Sourced from the SOPS nextcloud-bootstrap secret. ---
  - name: NEXTCLOUD_ADMIN_USER
    valueFrom:
      secretKeyRef:
        name: nextcloud-bootstrap
        key: admin-username
  - name: NEXTCLOUD_ADMIN_PASSWORD
    valueFrom:
      secretKeyRef:
        name: nextcloud-bootstrap
        key: admin-password
  # --- Public exposure behind the TLS-terminating shared Gateway (docs/08/09). TRUSTED_DOMAINS lets
  #     the vhost answer for the public FQDN; OVERWRITE* make Nextcloud emit https:// self-URLs even
  #     though it speaks plain HTTP behind the proxy. ---
  - name: NEXTCLOUD_TRUSTED_DOMAINS
    value: nextcloud.${BASE_DOMAIN}
  - name: OVERWRITEHOST
    value: nextcloud.${BASE_DOMAIN}
  - name: OVERWRITEPROTOCOL
    value: https
  - name: OVERWRITECLIURL
    value: https://nextcloud.${BASE_DOMAIN}
  # --- Keycloak native OIDC SSO (auth.mode oidc; docs/15 "Native OIDC"; R-IDP-02/06). The confidential
  #     `nextcloud` realm client's secret is mirrored into this namespace's nextcloud-bootstrap Secret
  #     (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The postStart hook reads it
  #     via user_oidc's --clientsecret-env (never on argv) to register the "keycloak" provider. The LOCAL
  #     admin above stays as the break-glass path (docs/15). ---
  - name: NEXTCLOUD_OIDC_CLIENT_SECRET
    valueFrom:
      secretKeyRef:
        name: nextcloud-bootstrap
        key: oidc-client-secret
lifecycle:
  postStart:
    exec:
      command:
        - /bin/sh
        - -c
        - |
          # Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint
          # (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR
          # updates the "keycloak" login provider (occ user_oidc:provider is create-or-update by
          # identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud
          # group provisioning. occ runs as www-data (owns config.php); the client secret is passed via
          # --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing
          # secret or not-yet-installed instance is skipped so the pod never crashloops (the local
          # break-glass admin still works).
          set -u
          # OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,
          # so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The
          # provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as
          # ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.
          OCC() { su -s /bin/sh www-data -c "php /var/www/html/occ $*"; }
          NEXTCLOUD_DISCOVERY_URL=https://keycloak.${BASE_DOMAIN}/realms/homelab/.well-known/openid-configuration
          nextcloud_oidc_fail() {
            echo "[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1" >&2
            { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true
          }
          # (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds
          #     maintenance mode during first-boot install/upgrade; occ writes made in that window do not
          #     stick — a second race on top of the quoting bug fixed at (c).
          i=0
          until OCC status 2>/dev/null | grep -q "installed: true" && ! OCC status 2>/dev/null | grep -q "maintenance: true"; do
            i=$((i + 1))
            [ "$i" -ge 300 ] && { echo "[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping" >&2; exit 0; }
            sleep 2
          done
          # Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak
          # FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's
          # SSRF guard otherwise REFUSES that request — OCP\Http\Client\LocalServerException "violates
          # local access rules" — during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to
          # the user as "Could not reach the provider". (The (b) curl check below uses system libcurl,
          # NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied
          # every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.
          OCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \
            || echo "[nextcloud-oidc] WARN: failed to set allow_local_remote_servers" >&2
          # NOTE the `$$` escape below — it is load-bearing. apps-wave-1 declares postBuild.
          # substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE
          # time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is
          # not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z "" ]` — ALWAYS
          # TRUE — so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never
          # installing/enabling user_oidc nor registering the provider. SSO kept working only because
          # the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with
          # NO OIDC at all. Flux STRICT substitution cannot catch this — it EXEMPTS default-operator
          # forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion
          # for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping — which is why
          # `--clientsecret-env` and the bare uses below always worked and only the guard broke.
          # Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.
          if [ -z "$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}" ]; then
            echo "[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)"
            exit 0
          fi
          if ! OCC app:list 2>/dev/null | grep -q "user_oidc"; then
            OCC app:install user_oidc || echo "[nextcloud-oidc] app:install user_oidc failed" >&2
          fi
          OCC app:enable user_oidc >/dev/null 2>&1 || true
          # (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.
          d=0; disc_ok=0
          while [ "$d" -lt 45 ]; do
            if [ "$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$NEXTCLOUD_DISCOVERY_URL" 2>/dev/null)" = "200" ]; then
              disc_ok=1; break
            fi
            d=$((d + 1)); sleep 2
          done
          [ "$disc_ok" = 1 ] || nextcloud_oidc_fail "Keycloak discovery endpoint not reachable (HTTP 200) within timeout"
          # (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.
          #     HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a
          #     single argument. The previous `OCC user_oidc:provider ... --scope="openid email profile"`
          #     form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->
          #     occ "Too many arguments" -> silent failure on EVERY boot. Secret via --clientsecret-env
          #     (never on argv). The verify-then-retry loop guarantees the provider actually persisted.
          r=0; reg_ok=0
          while [ "$r" -lt 10 ]; do
            su -s /bin/sh www-data -c "php /var/www/html/occ user_oidc:provider keycloak \
              --clientid=nextcloud \
              --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \
              --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \
              --scope='openid email profile' \
              --unique-uid=0 \
              --mapping-uid=preferred_username \
              --mapping-email=email \
              --mapping-display-name=name \
              --group-provisioning=1 \
              --mapping-groups=groups" >/dev/null 2>&1 || true
            if OCC user_oidc:provider 2>/dev/null | grep -q "keycloak"; then
              reg_ok=1; break
            fi
            r=$((r + 1)); sleep 3
          done
          if [ "$reg_ok" = 1 ]; then
            echo "[nextcloud-oidc] provider 'keycloak' configured and verified"
          else
            nextcloud_oidc_fail "user_oidc:provider ran but the provider did not persist after retries"
          fi
          # (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has
          #     NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a
          #     login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the
          #     built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every
          #     start: occ group:adduser is a no-op for an existing member. A user is promotable only
          #     AFTER their first OIDC login provisions them into the group, so a brand-new admin gains
          #     rights on their next login + the following pod start (or run `occ group:adduser admin
          #     <uid>` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is
          #     passed to php as argv (never interpolated) so group:list JSON parses robustly.
          ADMIN_SRC_GROUP=platform-admins
          # HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`
          # exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot
          # transient failure (DB / maintenance race in the first seconds after container start) silently
          # dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds
          # any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);
          # it is a no-op for members already in 'admin'. Group names are passed to php as argv (never
          # interpolated) so the JSON parses robustly.
          parse_group() { su -s /bin/sh www-data -c "php /var/www/html/occ group:list --output=json" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,"\n";}}' -- "$1" 2>/dev/null; }
          p=0
          while [ "$p" -lt 15 ]; do
            current_admins=$(parse_group admin)
            missing=0
            for uid in $(parse_group "$ADMIN_SRC_GROUP"); do
              if printf '%s\n' "$current_admins" | grep -qxF -- "$uid"; then
                continue
              fi
              missing=1
              if OCC group:adduser admin "$uid" >/dev/null 2>&1; then
                echo "[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)"
              fi
            done
            [ "$missing" = 0 ] && break
            p=$((p + 1)); sleep 2
          done
          if [ "$p" -ge 15 ]; then
            nextcloud_oidc_fail "could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries"
          fi
          # A postStart hook MUST exit 0. A non-zero exit makes the kubelet KILL the container
          # (FailedPostStartHook -> CrashLoopBackOff), which is exactly what happened on 2026-09-01:
          # the final statement used to be `[ "$p" -ge 15 ] && nextcloud_oidc_fail ...`, so on the
          # SUCCESS path the test returned 1 and became the script's exit status. That was latent for
          # as long as the OIDC guard above was dead code and exited 0 first; fixing the guard
          # exposed it immediately. Every failure path here logs loudly and writes a marker file and
          # deliberately does NOT crash the pod — the break-glass local admin must stay usable.
          exit 0
exposure: public                # edge port-forward + public DNS (docs/09; R-DNS/R-NET)
egress: internet                # normal WAN egress (docs/03; R-NET-05) — app store, integrity checks
storage:
  - name: data
    size: 20Gi
    class: longhorn             # ALL Nextcloud state (config/, data/, apps/, custom_apps/) on the
    mountPath: /var/www/html    # 3-replica tier; irreplaceable, DR-critical (docs/04; R-STATE-02).
                                # The whole html dir is persisted so config.php + user data survive a
                                # rebuild (the default /data/<name> would strand them on ephemeral rootfs).
dependencies:
  - to: nextcloud-db            # explicit allow: Nextcloud -> its PostgreSQL (docs/13 graph; R-NET-02).
    port: 5432                  # Renders the egress half; the DB namespace mirrors keycloak-db (no
    protocol: tcp               # default-deny), so Cilium's default-allow accepts the reciprocal ingress.
ingressRoutes:
  - host: nextcloud.${BASE_DOMAIN}
    port: 80                    # apache HTTP; wildcard TLS terminates at the shared Gateway (docs/08/09)
auth:
  mode: oidc                    # native Keycloak OIDC client (docs/13 SSO row, docs/15 "Native OIDC"; R-IDP-02)
  groups:
    - platform-admins           # access scoped to an authorized group (R-IDP-06)
  # WIRED (native OIDC): the Keycloak `nextcloud` confidential client (platform/keycloak/keycloak-realm-
  # import.yaml + live kcadm) is registered as a user_oidc login provider ("keycloak") by the postStart
  # hook (occ user_oidc:provider), using the client secret mirrored into nextcloud-bootstrap
  # (oidc-client-secret; same value SOPS-held in keycloak-realm-clients). The `groups` claim drives
  # user_oidc group provisioning (R-IDP-06). Members of `platform-admins` are ELEVATED to Nextcloud
  # admins by the postStart hook step (d) — user_oidc 8.11 has no native group->admin mapping, so the
  # provisioned membership is mirrored into the built-in `admin` group idempotently on every start; a
  # user is promoted after their first login provisions them + the next pod start. Nextcloud KEEPS its
  # LOCAL admin as the break-glass path so an IdP outage never locks the operator out (docs/15
  # "Availability caveat"). Reachability to the IdP at login requires allow_local_remote_servers=true
  # (set by the hook): the Keycloak FQDN resolves to the private gateway VIP, which Nextcloud's SSRF
  # guard blocks by default ("Could not reach the provider"). No secret is embedded here.
update:
  channel: stable               # R-UPD-01
  range: ">=31.0.0 <32.0.0"     # stay on the 31.x stable line
  cadence: "0 4 * * *"
  autoRollbackOnFailedHealth: true   # R-UPD-03
observability:
  metrics: true
  slis:
    - up                        # file-sync down is a critical alert (docs/10; R-OBS-01)
backup:
  policy: daily-offcluster      # DR-critical: data backed up off-cluster (docs/13, docs/04; R-STATE-03)