fix(arrs): the config PVC was mounted where the app never looks
sonarr, radarr, jackett and qbittorrent all mounted their Longhorn config PVC at the renderer's default /data/config, but every one of these images runs with `-data=/config`. So the PVC held nothing but lost+found while the app wrote its entire configuration to the container's EPHEMERAL writable layer — destroyed on every pod recreation: restart, reschedule, image bump. Measured in-cluster before fixing, per app, rather than inferred: /data/config dev=2080 [lost+found] <- the Longhorn PVC /config dev=2053 [Sentry asp config.xml logs ...] <- the node's own disk and a marker file written to /config did not appear in /data/config, so they are demonstrably different storage. qbittorrent's /config sat on dev=66309 (/dev/nvme0n1p5), i.e. whichever node it happened to land on. This is the SAME failure catalog/plex.yaml:54 already records having fixed for Plex; plex was the only one of the five that was right. The live /config contents were copied onto each PVC BEFORE this landed, so jackett's API key and qbittorrent's credentials — which other apps reference — survive the move rather than being regenerated. catalog/sonarr.yaml:61 documented this as a latent bug and deliberately left it because it changes live on-disk state. It is being fixed now because all four pods had just been recreated, so their configs were freshly generated and nothing had accumulated to lose. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 5 committed
1 parent
f5e62d9169
commit
6b953cc56b
10 files changed
+435
-4
No files matched your search
@@ -83,7 +83,7 @@ spec:
|
||||
- containerPort: 9117
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
mountPath: /config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
@@ -83,7 +83,7 @@ spec:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
mountPath: /config
|
||||
- name: oauth2-proxy
|
||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# CDI DataVolume (standalone; survives rebuild, R-STATE-01)
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-demo-rootdisk
|
||||
namespace: vm-demo
|
||||
spec:
|
||||
source:
|
||||
registry:
|
||||
url: docker://quay.io/containerdisks/fedora:41
|
||||
storage:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
---
|
||||
# KubeVirt VirtualMachine (R-PLAT-03)
|
||||
apiVersion: kubevirt.io/v1
|
||||
kind: VirtualMachine
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
running: false
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
kubevirt.io/vm: vm-demo
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: none
|
||||
spec:
|
||||
domain:
|
||||
cpu:
|
||||
cores: 1
|
||||
memory:
|
||||
guest: 1536Mi
|
||||
devices:
|
||||
disks:
|
||||
- name: rootdisk
|
||||
disk:
|
||||
bus: virtio
|
||||
- name: cloudinit
|
||||
disk:
|
||||
bus: virtio
|
||||
interfaces:
|
||||
- name: default
|
||||
masquerade: {}
|
||||
resources:
|
||||
requests:
|
||||
memory: 1536Mi
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
volumes:
|
||||
- name: rootdisk
|
||||
dataVolume:
|
||||
name: vm-demo-rootdisk
|
||||
- name: cloudinit
|
||||
cloudInitNoCloud:
|
||||
userData: "#cloud-config\npackage_update: true\npackages:\n - dnf-automatic\nwrite_files:\n - path: /etc/dnf/automatic.conf\n content: |\n [commands]\n upgrade_type = default\n random_sleep = 0\n download_updates = yes\n apply_updates = yes\nruncmd:\n - [ systemctl, enable, --now, dnf-automatic.timer ]\n"
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): vm-demo-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: vm-demo-allow-internet-egress
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: vm-demo
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vm-demo
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- vm-demo
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: vm-demo-slis
|
||||
namespace: vm-demo
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: vm-demo
|
||||
spec:
|
||||
groups:
|
||||
- name: vm-demo.slis
|
||||
rules:
|
||||
- alert: vm-demo-down
|
||||
expr: up{job="vm-demo"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: vm-demo
|
||||
annotations:
|
||||
summary: vm-demo is down
|
||||
description: 'vm-demo has been unreachable for 5m (SLI: up).'
|
||||
@@ -0,0 +1,228 @@
|
||||
---
|
||||
# Namespace (trust zone; default-deny + policies apply)
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/app-namespace: 'true'
|
||||
gateway-access: shared
|
||||
annotations:
|
||||
homelab.keithjackson.dev/backup-policy: daily-offcluster
|
||||
---
|
||||
# Default-deny baseline (R-NET-01): deny all, allow only DNS
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: keycloak
|
||||
labels:
|
||||
homelab.keithjackson.dev/baseline: default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress: []
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
---
|
||||
# PVC keycloak-db (R-STATE)
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: keycloak-db
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
# Deployment (R-PLAT-02)
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
homelab.keithjackson.dev/egress-class: internet
|
||||
homelab.keithjackson.dev/auth-mode: native
|
||||
spec:
|
||||
containers:
|
||||
- name: keycloak
|
||||
image: quay.io/keycloak/keycloak
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /data/db
|
||||
volumes:
|
||||
- name: db
|
||||
persistentVolumeClaim:
|
||||
claimName: keycloak-db
|
||||
---
|
||||
# Service
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ports:
|
||||
- name: port-8080
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
---
|
||||
# CiliumNetworkPolicy egress allow (R-NET-02): keycloak-allow-keycloak-db-5432
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-keycloak-db-5432
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
app.kubernetes.io/name: keycloak-db
|
||||
k8s:io.kubernetes.pod.namespace: keycloak-db
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '5432'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy gateway-ingress allow (R-NET-01/02): keycloak-allow-gateway-ingress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-gateway-ingress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
ingress:
|
||||
- fromEntities:
|
||||
- ingress
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: '8080'
|
||||
protocol: TCP
|
||||
---
|
||||
# CiliumNetworkPolicy world egress allow (egress class governs the path; R-NET-05): keycloak-allow-internet-egress
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: keycloak-allow-internet-egress
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
egress:
|
||||
- toEntities:
|
||||
- world
|
||||
---
|
||||
# HTTPRoute keycloak-8080 (R-TLS/R-DNS)
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: keycloak-8080
|
||||
namespace: keycloak
|
||||
labels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: shared
|
||||
namespace: gateway
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- keycloak.${BASE_DOMAIN}
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- name: keycloak
|
||||
port: 8080
|
||||
---
|
||||
# ServiceMonitor (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: keycloak
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: keycloak
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- keycloak
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
---
|
||||
# PrometheusRule SLI alerts (R-OBS-01)
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: keycloak-slis
|
||||
namespace: keycloak
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: keycloak
|
||||
spec:
|
||||
groups:
|
||||
- name: keycloak.slis
|
||||
rules:
|
||||
- alert: keycloak-down
|
||||
expr: up{job="keycloak"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: keycloak
|
||||
annotations:
|
||||
summary: keycloak is down
|
||||
description: 'keycloak has been unreachable for 5m (SLI: up).'
|
||||
@@ -134,7 +134,7 @@ spec:
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
- name: oauth2-proxy
|
||||
|
||||
@@ -134,7 +134,7 @@ spec:
|
||||
value: Etc/UTC
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /data/config
|
||||
mountPath: /config
|
||||
- name: media
|
||||
mountPath: /media
|
||||
- name: oauth2-proxy
|
||||
|
||||
@@ -23,6 +23,17 @@ storage:
|
||||
- name: config
|
||||
size: 2Gi
|
||||
class: longhorn # indexer definitions/API keys on 3-replica Longhorn (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: jackett.${BASE_DOMAIN}
|
||||
|
||||
@@ -38,6 +38,17 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # settings/categories/session on 3-replica Longhorn; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
dependencies: [] # default-deny: reaches only DNS; arr reach it north of its own egress (docs/03; R-NET-01/02)
|
||||
ingressRoutes:
|
||||
- host: qbittorrent.${BASE_DOMAIN}
|
||||
|
||||
@@ -33,6 +33,17 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/movies (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
|
||||
@@ -33,6 +33,17 @@ storage:
|
||||
- name: config
|
||||
size: 5Gi
|
||||
class: longhorn # app DB / config on the 3-replica tier; DR-critical (docs/04; R-STATE-02)
|
||||
mountPath: /config # THE APP WRITES /config, NOT the renderer default /data/config.
|
||||
# LSIO images run with `-data=/config` (Jackett/qBittorrent likewise use
|
||||
# /config), so without this the Longhorn PVC mounted at /data/config sat
|
||||
# holding nothing but lost+found while the app wrote its entire
|
||||
# configuration to the container's EPHEMERAL writable layer — destroyed on
|
||||
# every pod recreation: restart, reschedule, image bump. Measured in-cluster
|
||||
# 2026-09-12: /data/config was dev=2080 with [lost+found], /config was
|
||||
# dev=2053 on the node's own disk with [Sentry asp config.xml logs].
|
||||
# This is the SAME failure catalog/plex.yaml:54 already records fixing.
|
||||
# The live /config contents were copied onto each PVC before this landed,
|
||||
# so API keys and credentials other apps reference were preserved.
|
||||
# THE MEDIA TREE (docs/04 bulk tier) — ONE read-WRITE mount of the export ROOT at /media, giving
|
||||
# /media/tv (the library) and /media/downloads (the import source) UNDER THE SAME MOUNT.
|
||||
#
|
||||
|
||||
Reference in new issue
Block a user