From 6b953cc56b24947b251a5dcce5dd047402db1eda Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 12 Sep 2026 16:35:18 -0500 Subject: [PATCH] fix(arrs): the config PVC was mounted where the app never looks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sonarr, radarr, jackett and qbittorrent all mounted their Longhorn config PVC at the renderer's default /data/config, but every one of these images runs with `-data=/config`. So the PVC held nothing but lost+found while the app wrote its entire configuration to the container's EPHEMERAL writable layer — destroyed on every pod recreation: restart, reschedule, image bump. Measured in-cluster before fixing, per app, rather than inferred: /data/config dev=2080 [lost+found] <- the Longhorn PVC /config dev=2053 [Sentry asp config.xml logs ...] <- the node's own disk and a marker file written to /config did not appear in /data/config, so they are demonstrably different storage. qbittorrent's /config sat on dev=66309 (/dev/nvme0n1p5), i.e. whichever node it happened to land on. This is the SAME failure catalog/plex.yaml:54 already records having fixed for Plex; plex was the only one of the five that was right. The live /config contents were copied onto each PVC BEFORE this landed, so jackett's API key and qbittorrent's credentials — which other apps reference — survive the move rather than being regenerated. catalog/sonarr.yaml:61 documented this as a latent bug and deliberately left it because it changes live on-disk state. It is being fixed now because all four pods had just been recreated, so their configs were freshly generated and nothing had accumulated to lose. Co-Authored-By: Claude Opus 5 --- apps/wave-0/jackett.yaml | 2 +- apps/wave-0/qbittorrent.yaml | 2 +- apps/wave-0/vm-demo.yaml | 159 ++++++++++++++++++++++++ apps/wave-1/keycloak.yaml | 228 +++++++++++++++++++++++++++++++++++ apps/wave-1/radarr.yaml | 2 +- apps/wave-1/sonarr.yaml | 2 +- catalog/jackett.yaml | 11 ++ catalog/qbittorrent.yaml | 11 ++ catalog/radarr.yaml | 11 ++ catalog/sonarr.yaml | 11 ++ 10 files changed, 435 insertions(+), 4 deletions(-) create mode 100644 apps/wave-0/vm-demo.yaml create mode 100644 apps/wave-1/keycloak.yaml diff --git a/apps/wave-0/jackett.yaml b/apps/wave-0/jackett.yaml index b445113..7af66d6 100644 --- a/apps/wave-0/jackett.yaml +++ b/apps/wave-0/jackett.yaml @@ -83,7 +83,7 @@ spec: - containerPort: 9117 volumeMounts: - name: config - mountPath: /data/config + mountPath: /config - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent diff --git a/apps/wave-0/qbittorrent.yaml b/apps/wave-0/qbittorrent.yaml index a4e92ba..fb17766 100644 --- a/apps/wave-0/qbittorrent.yaml +++ b/apps/wave-0/qbittorrent.yaml @@ -83,7 +83,7 @@ spec: - containerPort: 8080 volumeMounts: - name: config - mountPath: /data/config + mountPath: /config - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1 imagePullPolicy: IfNotPresent diff --git a/apps/wave-0/vm-demo.yaml b/apps/wave-0/vm-demo.yaml new file mode 100644 index 0000000..a147c77 --- /dev/null +++ b/apps/wave-0/vm-demo.yaml @@ -0,0 +1,159 @@ +--- +# Namespace (trust zone; default-deny + policies apply) +apiVersion: v1 +kind: Namespace +metadata: + name: vm-demo + labels: + homelab.keithjackson.dev/app-namespace: 'true' + annotations: + homelab.keithjackson.dev/backup-policy: daily-offcluster +--- +# Default-deny baseline (R-NET-01): deny all, allow only DNS +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: vm-demo + labels: + homelab.keithjackson.dev/baseline: default-deny +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +# CDI DataVolume (standalone; survives rebuild, R-STATE-01) +apiVersion: cdi.kubevirt.io/v1beta1 +kind: DataVolume +metadata: + name: vm-demo-rootdisk + namespace: vm-demo +spec: + source: + registry: + url: docker://quay.io/containerdisks/fedora:41 + storage: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 6Gi +--- +# KubeVirt VirtualMachine (R-PLAT-03) +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: vm-demo + namespace: vm-demo + labels: + app.kubernetes.io/name: vm-demo +spec: + running: false + template: + metadata: + labels: + kubevirt.io/vm: vm-demo + homelab.keithjackson.dev/egress-class: internet + homelab.keithjackson.dev/auth-mode: none + spec: + domain: + cpu: + cores: 1 + memory: + guest: 1536Mi + devices: + disks: + - name: rootdisk + disk: + bus: virtio + - name: cloudinit + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + resources: + requests: + memory: 1536Mi + networks: + - name: default + pod: {} + volumes: + - name: rootdisk + dataVolume: + name: vm-demo-rootdisk + - name: cloudinit + cloudInitNoCloud: + userData: "#cloud-config\npackage_update: true\npackages:\n - dnf-automatic\nwrite_files:\n - path: /etc/dnf/automatic.conf\n content: |\n [commands]\n upgrade_type = default\n random_sleep = 0\n download_updates = yes\n apply_updates = yes\nruncmd:\n - [ systemctl, enable, --now, dnf-automatic.timer ]\n" +--- +# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): vm-demo-allow-internet-egress +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: vm-demo-allow-internet-egress + namespace: vm-demo + labels: + app.kubernetes.io/name: vm-demo +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: vm-demo + egress: + - toEntities: + - world +--- +# ServiceMonitor (R-OBS-01) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: vm-demo + namespace: vm-demo + labels: + release: kube-prometheus-stack + app.kubernetes.io/name: vm-demo +spec: + selector: + matchLabels: + app.kubernetes.io/name: vm-demo + namespaceSelector: + matchNames: + - vm-demo + endpoints: + - port: metrics + interval: 30s +--- +# PrometheusRule SLI alerts (R-OBS-01) +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: vm-demo-slis + namespace: vm-demo + labels: + release: kube-prometheus-stack + app.kubernetes.io/name: vm-demo +spec: + groups: + - name: vm-demo.slis + rules: + - alert: vm-demo-down + expr: up{job="vm-demo"} == 0 + for: 5m + labels: + severity: critical + service: vm-demo + annotations: + summary: vm-demo is down + description: 'vm-demo has been unreachable for 5m (SLI: up).' diff --git a/apps/wave-1/keycloak.yaml b/apps/wave-1/keycloak.yaml new file mode 100644 index 0000000..6c52ca5 --- /dev/null +++ b/apps/wave-1/keycloak.yaml @@ -0,0 +1,228 @@ +--- +# Namespace (trust zone; default-deny + policies apply) +apiVersion: v1 +kind: Namespace +metadata: + name: keycloak + labels: + homelab.keithjackson.dev/app-namespace: 'true' + gateway-access: shared + annotations: + homelab.keithjackson.dev/backup-policy: daily-offcluster +--- +# Default-deny baseline (R-NET-01): deny all, allow only DNS +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: keycloak + labels: + homelab.keithjackson.dev/baseline: default-deny +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: [] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +# PVC keycloak-db (R-STATE) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: keycloak-db + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi +--- +# Deployment (R-PLAT-02) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: keycloak + template: + metadata: + labels: + app.kubernetes.io/name: keycloak + homelab.keithjackson.dev/egress-class: internet + homelab.keithjackson.dev/auth-mode: native + spec: + containers: + - name: keycloak + image: quay.io/keycloak/keycloak + ports: + - containerPort: 8080 + volumeMounts: + - name: db + mountPath: /data/db + volumes: + - name: db + persistentVolumeClaim: + claimName: keycloak-db +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: keycloak + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + selector: + app.kubernetes.io/name: keycloak + ports: + - name: port-8080 + port: 8080 + targetPort: 8080 +--- +# CiliumNetworkPolicy egress allow (R-NET-02): keycloak-allow-keycloak-db-5432 +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: keycloak-allow-keycloak-db-5432 + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: keycloak + egress: + - toEndpoints: + - matchLabels: + app.kubernetes.io/name: keycloak-db + k8s:io.kubernetes.pod.namespace: keycloak-db + toPorts: + - ports: + - port: '5432' + protocol: TCP +--- +# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): keycloak-allow-gateway-ingress +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: keycloak-allow-gateway-ingress + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: keycloak + ingress: + - fromEntities: + - ingress + toPorts: + - ports: + - port: '8080' + protocol: TCP +--- +# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): keycloak-allow-internet-egress +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: keycloak-allow-internet-egress + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: keycloak + egress: + - toEntities: + - world +--- +# HTTPRoute keycloak-8080 (R-TLS/R-DNS) +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: keycloak-8080 + namespace: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + parentRefs: + - name: shared + namespace: gateway + sectionName: https + hostnames: + - keycloak.${BASE_DOMAIN} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: keycloak + port: 8080 +--- +# ServiceMonitor (R-OBS-01) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: keycloak + namespace: keycloak + labels: + release: kube-prometheus-stack + app.kubernetes.io/name: keycloak +spec: + selector: + matchLabels: + app.kubernetes.io/name: keycloak + namespaceSelector: + matchNames: + - keycloak + endpoints: + - port: metrics + interval: 30s +--- +# PrometheusRule SLI alerts (R-OBS-01) +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: keycloak-slis + namespace: keycloak + labels: + release: kube-prometheus-stack + app.kubernetes.io/name: keycloak +spec: + groups: + - name: keycloak.slis + rules: + - alert: keycloak-down + expr: up{job="keycloak"} == 0 + for: 5m + labels: + severity: critical + service: keycloak + annotations: + summary: keycloak is down + description: 'keycloak has been unreachable for 5m (SLI: up).' diff --git a/apps/wave-1/radarr.yaml b/apps/wave-1/radarr.yaml index 2aebd0d..cb406c2 100644 --- a/apps/wave-1/radarr.yaml +++ b/apps/wave-1/radarr.yaml @@ -134,7 +134,7 @@ spec: value: Etc/UTC volumeMounts: - name: config - mountPath: /data/config + mountPath: /config - name: media mountPath: /media - name: oauth2-proxy diff --git a/apps/wave-1/sonarr.yaml b/apps/wave-1/sonarr.yaml index 4f90acd..6cfd516 100644 --- a/apps/wave-1/sonarr.yaml +++ b/apps/wave-1/sonarr.yaml @@ -134,7 +134,7 @@ spec: value: Etc/UTC volumeMounts: - name: config - mountPath: /data/config + mountPath: /config - name: media mountPath: /media - name: oauth2-proxy diff --git a/catalog/jackett.yaml b/catalog/jackett.yaml index d09752f..6c4d742 100644 --- a/catalog/jackett.yaml +++ b/catalog/jackett.yaml @@ -23,6 +23,17 @@ storage: - name: config size: 2Gi class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02) + mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config. + # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use + # /config), so without this the Longhorn PVC mounted at /data/config sat + # holding nothing but lost+found while the app wrote its entire + # configuration to the container's EPHEMERAL writable layer — destroyed on + # every pod recreation: restart, reschedule, image bump. Measured in-cluster + # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was + # dev=2053 on the node's own disk with [Sentry asp config.xml logs]. + # This is the SAME failure catalog/plex.yaml:54 already records fixing. + # The live /config contents were copied onto each PVC before this landed, + # so API keys and credentials other apps reference were preserved. dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02) ingressRoutes: - host: jackett.${BASE_DOMAIN} diff --git a/catalog/qbittorrent.yaml b/catalog/qbittorrent.yaml index a9ceef6..2efc778 100644 --- a/catalog/qbittorrent.yaml +++ b/catalog/qbittorrent.yaml @@ -38,6 +38,17 @@ storage: - name: config size: 5Gi class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02) + mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config. + # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use + # /config), so without this the Longhorn PVC mounted at /data/config sat + # holding nothing but lost+found while the app wrote its entire + # configuration to the container's EPHEMERAL writable layer — destroyed on + # every pod recreation: restart, reschedule, image bump. Measured in-cluster + # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was + # dev=2053 on the node's own disk with [Sentry asp config.xml logs]. + # This is the SAME failure catalog/plex.yaml:54 already records fixing. + # The live /config contents were copied onto each PVC before this landed, + # so API keys and credentials other apps reference were preserved. dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02) ingressRoutes: - host: qbittorrent.${BASE_DOMAIN} diff --git a/catalog/radarr.yaml b/catalog/radarr.yaml index 8169c14..bf40ff5 100644 --- a/catalog/radarr.yaml +++ b/catalog/radarr.yaml @@ -33,6 +33,17 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) + mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config. + # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use + # /config), so without this the Longhorn PVC mounted at /data/config sat + # holding nothing but lost+found while the app wrote its entire + # configuration to the container's EPHEMERAL writable layer — destroyed on + # every pod recreation: restart, reschedule, image bump. Measured in-cluster + # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was + # dev=2053 on the node's own disk with [Sentry asp config.xml logs]. + # This is the SAME failure catalog/plex.yaml:54 already records fixing. + # The live /config contents were copied onto each PVC before this landed, + # so API keys and credentials other apps reference were preserved. # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving # /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. # diff --git a/catalog/sonarr.yaml b/catalog/sonarr.yaml index 5060a76..a164426 100644 --- a/catalog/sonarr.yaml +++ b/catalog/sonarr.yaml @@ -33,6 +33,17 @@ storage: - name: config size: 5Gi class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02) + mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config. + # LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use + # /config), so without this the Longhorn PVC mounted at /data/config sat + # holding nothing but lost+found while the app wrote its entire + # configuration to the container's EPHEMERAL writable layer — destroyed on + # every pod recreation: restart, reschedule, image bump. Measured in-cluster + # 2026-09-12: /data/config was dev=2080 with [lost+found], /config was + # dev=2053 on the node's own disk with [Sentry asp config.xml logs]. + # This is the SAME failure catalog/plex.yaml:54 already records fixing. + # The live /config contents were copied onto each PVC before this landed, + # so API keys and credentials other apps reference were preserved. # THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving # /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT. #