home-network-k3s-impl/catalog/pihole.yaml

# ServiceDefinition: pihole
# Source of truth: spec docs/13-service-catalog.md, row 6.
# internal | egress internet | config on Longhorn (DR-critical config) | auth forward-auth.
# Also a platform DNS dependency (docs/09). Validates against schemas/servicedefinition.schema.json.
name: pihole
kind: container
source:
  image: pihole/pihole
exposure: internal            # LAN-only admin UI (docs/13 row 6; R-DNS/R-NET)
egress: internet              # normal WAN egress (docs/03; R-NET-05)
storage:
  - name: config
    size: 2Gi
    class: longhorn           # replicated config state; DR-critical (docs/04 line 118; R-STATE-02/03)
    # THE PVC WAS MOUNTED SOMEWHERE PIHOLE NEVER WRITES, AND SO HELD NOTHING AT ALL. Without an
    # explicit mountPath the renderer defaults to /data/<name>, i.e. /data/config — but PiHole keeps
    # every piece of its state in /etc/pihole. MEASURED on live prod 2026-10-04: `/etc/pihole` (7.1 M
    # — gravity.db, pihole.toml, the query DB) was on the **overlay** filesystem, i.e. the
    # container's EPHEMERAL layer, while `/dev/longhorn/pvc-849edb86...` sat mounted at
    # /data/config holding `lost+found` and 24 K of nothing. Three consequences, all live until now:
    #   * every restart silently discarded PiHole's whole state, re-downloading 72 k gravity domains
    #     and regenerating the admin password hash;
    #   * `backup.policy: daily-offcluster` below has been faithfully backing up an EMPTY volume,
    #     which is worse than no backup because it reports success;
    #   * it is also why phase 1's password/adlist declaration mattered more than it first appeared —
    #     that state was not merely undeclared, it was ephemeral.
    # This is the same class of bug catalog/plex.yaml's `config` entry documents ("Without this the
    # PVC sat unused and Plex wrote to the container's EPHEMERAL layer").
    #
    # /etc/pihole IS THE IMAGE'S INTENDED MOUNT POINT, and an EMPTY volume there self-initialises.
    # Verified in the running image, not assumed:
    #   * `/usr/bin/start.sh` (PID 1) calls `migrate_gravity`, whose `migrate_to_database()` creates
    #     the gravity database when `[ ! -e "$gravityDBfile" ]` — the same branch the postStart hook
    #     below relies on, and the branch that produced the current ephemeral copy.
    #   * FTL writes `pihole.toml` itself from the `FTLCONF_*` env above (that is what the
    #     `### CHANGED (env)` markers in it are).
    #   * NO `podSecurityContext.fsGroup` is needed and adding one would be inert: `start.sh` line 5
    #     is `chown -R pihole:pihole /etc/pihole/ /var/log/pihole/` and PID 1 runs as **uid 0**, so
    #     the image fixes ownership of a fresh root-owned volume before anything else starts. Same
    #     arrangement as plex's image managing its own /config ownership.
    #   * the image's own default for `files.macvendor` was moved OUT of this directory (to
    #     `/macvendor.db`), which is what a maintainer does to a path they expect to be a volume.
    #
    # WHY IT IS FIXED IN THIS BRANCH rather than left for its own: `availability.replicas: 2` renders
    # per-replica volumeClaimTemplates, and with the wrong mountPath those would be two 2 Gi Longhorn
    # volumes (times three Longhorn replicas each) holding nothing but `lost+found` while all state
    # stayed ephemeral — paying the cost of persistence and getting none of it. It also REMOVES this
    # change's main operational risk: with gravity.db persisted, a pod restart is cheap, instead of
    # every restart re-downloading 72 k domains inside a wave that has a 5 m timeout.
    mountPath: /etc/pihole
dependencies: []              # default-deny: reaches only DNS; it PROVIDES DNS (docs/03; R-NET-01/02)
# ------------------------------------------------------------------------------------------------
# LAN SPLIT-HORIZON DNS — the half of R-DNS-03 that platform/coredns/coredns-custom.yaml defers to.
# ------------------------------------------------------------------------------------------------
# coredns-custom answers the ${BASE_DOMAIN} zone for PODS ONLY and says so in its own header ("LAN
# clients keep their own answers (PiHole split-horizon)"). Until now PiHole served no DNS at all —
# its Service was ClusterIP :80/:4180, no port 53, no LoadBalancer — so the LAN half did not exist
# and LAN clients depended on hand-maintained per-client dnsmasq files. One went stale after the dev
# decommission and cost the operator sonarr.${BASE_DOMAIN}. Upstream cannot cover for it either: the
# SOHO router that owns this LAN strips RFC1918 answers for public domains as rebind protection and
# returns NOERROR / ANSWER 0. apps/wave-0/pihole-dns.yaml exposes udp/53 + tcp/53 on
# ${PIHOLE_DNS_VIP}; these env vars are what make the ANSWERS declarative.
#
# WHY env AND NOT A RECONCILER: the repo's level-triggered CronJob reconcilers
# (platform/keycloak/client-secret-reconciler.yaml, the CNPG grant reconciler in
# apps/wave-0/nextcloud-db.yaml) exist because their state has NO declarative entry point — a
# Keycloak client secret lives in Keycloak's own database and a Postgres GRANT lives in the
# catalog, and nothing in a manifest can express either. PiHole v6 is not that case. Every FTL
# setting has an `FTLCONF_<section>_<key>` environment variable, and a value supplied that way is
# ENV-FORCED: FTL logs it as "forced through environment", applies it on every start ahead of
# /etc/pihole/pihole.toml, and marks it read-only in the admin UI so a human cannot drift it. That
# is strictly better than a reconciler — it is enforced at t=0 on a cold start rather than at the
# next cron tick, it cannot be defeated by the UI, and it carries no extra RBAC, image or Job. A
# CronJob here would be ceremony around a knob the app already exposes. (Verified on the live prod
# image, PiHole Core v6.4.3 / FTL v6.7, 2026-09-14.)
#
# THE PVC IS NOT THE MECHANISM AND MUST NOT BE. The `config` volume below mounts at /data/config;
# PiHole's actual state directory is /etc/pihole, which is EPHEMERAL in this Deployment. Anything
# configured through the UI is lost on the next reschedule — which is exactly what happened to the
# upstream servers: the live pod's pihole.toml carried a hand-set `upstreams` marked "### CHANGED"
# that no manifest declared, so a restart would have left PiHole with upstreams = [] and no
# recursion at all. Declaring upstreams here closes that too: PiHole must stay a usable GENERAL
# resolver, not only authoritative for this zone, or pointing LAN clients at it breaks the internet.
env:
  # Default is LOCAL, which answers only clients on the same subnet as the listening interface. The
  # pod's interface is on the CLUSTER pod CIDR, so every LAN client would be refused. ALL is
  # required for a LoadBalancer-fronted resolver; what keeps PiHole from being an OPEN resolver is
  # the exposure, not this setting — `loadBalancerSourceRanges: [${LAN_CIDR}]` on the DNS Services,
  # enforced by Cilium in the datapath before load-balancing, plus a CiliumNetworkPolicy that admits
  # nothing but udp/53 and tcp/53. See apps/wave-0/pihole-dns.yaml.
  - name: FTLCONF_dns_listeningMode
    value: "ALL"
  # Recursion for everything that is NOT this zone. Public resolvers on purpose, NOT the SOHO
  # router at the LAN gateway: that router is the box whose rebind protection strips RFC1918
  # answers, and forwarding to a device that may forward back here invites a loop.
  - name: FTLCONF_dns_upstreams
    value: "8.8.8.8;8.8.4.4"
  # Per-client query rate limit OFF (default: 1000 queries per 60s, then REFUSED). It has to be off
  # because of how the DNS Services are exposed: apps/wave-0/pihole-dns.yaml uses
  # externalTrafficPolicy Cluster — it must, or LAN resolution breaks whenever Cilium's L2
  # announcement lease and the pihole pod land on different nodes, which is most of the time — and
  # Cilium runs in SNAT mode here, so a query forwarded between nodes reaches PiHole carrying the
  # INGRESS NODE's address instead of the client's. Every LAN device therefore counts against ONE
  # apparent client's budget, and a normal evening of browsing from a handful of machines would trip
  # a limit meant to catch one misbehaving host. Leaving it at the default turns the whole LAN's DNS
  # off for 60 seconds at a time, intermittently, which is the worst possible failure shape. The
  # limit becomes meaningful again — and should be restored — if Cilium is ever switched to
  # `loadBalancer.mode: dsr`, which preserves the client address.
  - name: FTLCONF_dns_rateLimit_count
    value: "0"
  # THE SPLIT-HORIZON ANSWERS. A dnsmasq `address=/<domain>/<ip>` entry answers that domain AND
  # every subdomain of it, and dnsmasq resolves overlapping entries by LONGEST MATCHING DOMAIN —
  # so the first line is a wildcard for the whole zone and each later line is a specific override.
  # PRECEDENCE IS THEREFORE STRUCTURAL, not order-dependent: `unifi.${BASE_DOMAIN}` matches both
  # line 1 and line 2, line 2 is the longer domain, line 2 wins. Rewriting these in any order
  # changes nothing.
  #
  # A NEWLY ONBOARDED SERVICE NEEDS NO CHANGE HERE. Every catalog ingressRoute host is
  # `<label>.${BASE_DOMAIN}` served by the shared Gateway, so the wildcard already answers it with
  # ${GATEWAY_VIP} — the same guarantee, and the same reasoning, as the `template IN A` block in
  # platform/coredns/coredns-custom.yaml. Add a line here ONLY for a service that does not live
  # behind the Gateway: an L4 LoadBalancer on its own pinned VIP. Those are enumerated from the
  # repo, not from memory — grep `external-dns.alpha.kubernetes.io/hostname` on a Service, which
  # is the annotation that means "this Service publishes its own hostname at its own address":
  # apps/vms/unifi/50-service-l4.yaml (unifi -> ${UNIFI_L4_VIP}) and
  # platform/crafty/service-l4.yaml (crafty-mc -> ${CRAFTY_L4_VIP}). Those two, plus PiHole's own
  # resolver address, are the complete set of exceptions today.
  #
  # AAAA needs nothing: dnsmasq answers a AAAA query for an `address=`d name with NODATA
  # (NOERROR, no answer), so IPv6-capable clients fall back to the A record instead of getting
  # NXDOMAIN — the same behaviour coredns-custom hand-writes a `template IN AAAA` block for.
  #
  # Array-valued FTLCONF variables are SEMICOLON-separated. A JSON-looking list is accepted and
  # then silently wrong: FTL stores `["a","b"]` as ONE array element containing those literal
  # characters, writes that single bogus line into /etc/pihole/dnsmasq.conf and reports "0 invalid"
  # while resolution for the zone does not work. Confirmed both ways against the live image.
  # plex-vip -> ${PLEX_L4_VIP} (added 2026-10-04) is the LAN HALF of Plex's single advertised
  # hostname. Plex publishes ONE URL to plex.tv for both local and remote clients
  # (catalog/plex.yaml ADVERTISE_IP): on the LAN this answer points it at the tcp/32400
  # LoadBalancer, while Cloudflare answers the same name publicly with the WAN address, NATed to
  # the same VIP. WITHOUT THIS LINE the entry above answers it with ${GATEWAY_VIP} via the zone
  # wildcard, where NOTHING listens on 32400 — measured 2026-10-04, plex-vip.${BASE_DOMAIN}
  # resolved to the Gateway VIP and a connect to :32400 there returned http=000. That is a silent
  # failure: the Plex web UI on plex.${BASE_DOMAIN} keeps working through the Gateway on 443 while
  # every client that takes the ADVERTISED connection cannot reach the server at all.
  - name: FTLCONF_misc_dnsmasq_lines
    value: "address=/${BASE_DOMAIN}/${GATEWAY_VIP};address=/unifi.${BASE_DOMAIN}/${UNIFI_L4_VIP};address=/crafty-mc.${BASE_DOMAIN}/${CRAFTY_L4_VIP};address=/pihole-dns.${BASE_DOMAIN}/${PIHOLE_DNS_VIP};address=/plex-vip.${BASE_DOMAIN}/${PLEX_L4_VIP}"
  # THE ADMIN PASSWORD, previously the ONE operator-set value declared nowhere. Sourced from
  # apps/wave-0/pihole-bootstrap-secret.sops.yaml (SOPS/age, same wave => same reconcile), which
  # carries the full rationale and the measurement that found it.
  #
  # `webserver.api.password` takes CLEARTEXT and FTL hashes it itself into `webserver.api.pwhash`.
  # Verified in the running image, not assumed: `pihole setpassword` is itself
  # `setFTLConfigValue "webserver.api.password" "$PASSWORD"`, commented upstream as "pihole-FTL will
  # automatically hash the password". So the BALLOON-SHA256 hash on the PVC is now a DERIVED artefact
  # and this is its source of truth. Rotating the Secret rotates the password.
  - name: FTLCONF_webserver_api_password
    valueFrom:
      secretKeyRef:
        name: pihole-bootstrap
        key: webui-password
availability:
  # TWO REPLICAS, ON DIFFERENT NODES. LAN DNS is a critical network service and it was ONE pod on
  # ONE node. Measured 2026-10-04, that cost more than "until it reschedules":
  #   * the node running the pod dies -> ~40s to mark it NotReady, then `tolerationSeconds: 300`
  #     before the pod is even evicted, then reschedule, then Longhorn must force-detach the RWO
  #     volume from a dead node and reattach it. Roughly 6-10 MINUTES with no LAN resolution.
  #   * the node holding the `pihole-dns-*` L2 lease dies -> Cilium re-elects an announcer, tens of
  #     seconds. (These were DIFFERENT nodes: lease on k3s-prod-2, pod on k3s-prod-1. Two of three
  #     nodes were single points of failure for LAN DNS, with wildly different blast radii.)
  # The Services already tolerate this: `externalTrafficPolicy: Cluster` on the DNS LoadBalancers
  # (apps/wave-0/pihole-dns.yaml) means the L2 lease holder forwards to ANY ready backend, so a
  # second replica is picked up with no change there.
  #
  # WHAT THIS DOES NOT FIX, and it is the bigger risk: LAN DNS still depends on the CLUSTER. etcd
  # quorum loss, a Cilium regression, a Longhorn problem or a bad reconcile takes DNS down
  # everywhere regardless of replica count. The only thing that removes that dependency is a
  # resolver outside the cluster carrying the same `address=` lines. Considered and not chosen
  # (operator decision, 2026-10-04); see decisions/2026-10-04-pihole-multi-replica.md.
  #
  # THE PREREQUISITE WAS DONE FIRST, and this field would have been actively harmful without it. A
  # replica gets a FRESH volume, and until 2026-10-04 PiHole's admin password and adlists lived only
  # on the PVC — so replica 1 would have come up SERVING DNS AND BLOCKING NOTHING, with a different
  # password, and answers would have varied by which pod replied. Both are now declared (the
  # password as a SOPS Secret -> FTLCONF_webserver_api_password; the adlists reconciled into
  # gravity.db by the postStart hook below), which is what makes a fresh replica identical to its
  # sibling. See decisions/2026-10-04-pihole-declare-state.md.
  #
  # WHY 2 AND NOT 3. Two tolerates the loss of any single node, which is the stated goal, and leaves
  # a spare node for the evicted replica to land on under the HARD anti-affinity the renderer emits.
  # Three would pin one replica per node with no spare, so a single node failure would leave a
  # replica permanently Pending and — `apps-wave-0` being `wait: true` — stall the app tier for as
  # long as the node was down. Two is the count that degrades gracefully here.
  replicas: 2
ingressRoutes:
  - host: pihole.${BASE_DOMAIN}
    port: 80                  # admin web UI, protected at the Gateway by forward-auth (docs/09)
auth:
  mode: forward-auth          # oauth2-proxy -> Keycloak; bare UI, no native OIDC (docs/15; R-IDP-02)
  groups:
    - platform-admins         # sensitive admin UI: authenticated AND in group (R-IDP-06)
  # NO `apiRoutes` HERE, DELIBERATELY. PiHole's admin UI serves no /api/ path, so the `^/api/`
  # that jackett and radarr need would match nothing on a good day — and a BROADER guess would
  # answer PiHole's own page loads with 401, leaving a logged-out operator on a bare error page
  # with no sign-in offered. This app is the reason auth.apiRoutes is per-service (schema 1.9.0)
  # rather than a renderer constant: the measurement that justifies the flag on two apps is
  # evidence about those two apps. Its oauth2-proxy log holds 0 unauthenticated requests across
  # the current pod's whole 23h lifetime (measured 2026-10-01), so there is nothing to act on.
lifecycle:
  # LEVEL-TRIGGERED ADLIST ASSERTION, run on EVERY container start. Same pattern, and the same
  # reasoning, as catalog/qbittorrent.yaml's postStart: desired state lives in git and is
  # RE-ASSERTED unconditionally rather than set once by hand.
  #
  # WHY THIS IS NEEDED AT ALL — the blocklists were NOT reproducible from git. Measured on live prod
  # 2026-10-04: `gravity.db` held exactly ONE adlist (StevenBlack's hosts, enabled) and 72,234
  # gravity domains, and NOTHING in this repo named it. Worse, a FRESH database does not seed it:
  # the image's own schema template `/etc/.pihole/advanced/Templates/gravity.db.sql` creates the
  # `adlist` table and inserts NO rows into it (grepped in the running image — there is no
  # `INSERT INTO adlist` at all). So a rebuild from git, or a replica with a new PVC, would come up
  # SERVING DNS AND BLOCKING NOTHING, silently and with every object reporting healthy.
  #
  # WHY A postStart HOOK AND NOT A FILE. PiHole v6 keeps adlists as DB rows, not config. The
  # `/etc/pihole/adlists.list` file still exists in `gravity.sh` but is MIGRATION-ONLY: its import
  # sits inside `migrate_to_database()`, guarded by `if [ ! -e "$gravityDBfile" ]`, so it fires only
  # while CREATING a database and never again on an existing one. A ConfigMap mounted there would
  # therefore be silently ignored on every pod this cluster currently runs. There is also no
  # `pihole adlist` CLI subcommand (only allow/deny for domains), so SQL against the DB is the
  # actual interface. FTL ships an embedded shell (`pihole-FTL sqlite3`, SQLite 3.53.1) so no extra
  # tooling is needed in the image.
  #
  # IT RECONCILES, IT DOES NOT JUST ADD: adlists present in the DB but absent from DECLARED below are
  # DISABLED (not deleted — deleting cascades to `gravity` rows and would force a full re-download of
  # everything on a typo). So this list is the source of truth in both directions, and `pihole -g`
  # runs ONLY when something actually changed, which keeps the steady-state start free of a 70k-domain
  # rebuild.
  #
  # NO BRACED DOLLAR EXPANSION ANYWHERE IN THIS SCRIPT, DELIBERATELY — the same contract
  # catalog/qbittorrent.yaml documents at length. apps-wave-0 declares postBuild.substituteFrom
  # cluster-vars, so Flux runs envsubst over this whole manifest at reconcile time; a `${VAR}` form
  # would behave differently on the cluster than under a plain `kubectl apply` of the golden. Bare
  # `$VAR` and `$(cmd)` are untouched by envsubst and so behave identically however this is run.
  # `render.py`'s assert_substitution_safe refuses to emit a manifest that breaks this.
  postStart:
    exec:
      command:
        - /bin/bash
        - -c
        - |
          # Assert the declared adlists in gravity.db. Idempotent: steady state writes nothing and
          # does NOT run a gravity rebuild.
          set -u
          GRAVITY=/etc/pihole/gravity.db

          # THE DECLARED SET — one URL per line, the source of truth for what PiHole blocks.
          # To add a list: add a line here, commit, let Flux reconcile. Nothing else to do.
          DECLARED='https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'

          # On a fresh PVC, FTL creates gravity.db a moment after the container starts. Wait, bounded,
          # and NEVER block or fail the start: a missing DB here must not crashloop DNS for the LAN.
          i=0
          while [ ! -s "$GRAVITY" ] && [ "$i" -lt 90 ]; do sleep 2; i=$(expr $i + 1); done
          if [ ! -s "$GRAVITY" ]; then
            echo "pihole-adlists: gravity.db absent after 180s - leaving adlists untouched" >&2
            exit 0
          fi

          sql() { pihole-FTL sqlite3 "$GRAVITY" "$1"; }
          CHANGED=0

          # 1) Every declared URL must be present AND enabled.
          echo "$DECLARED" | while IFS= read -r url; do
            [ -n "$url" ] || continue
            printf '%s\n' "$url"
          done > /tmp/declared.txt
          while IFS= read -r url; do
            [ -n "$url" ] || continue
            # Single quotes inside a URL would break this SQL; adlist URLs do not contain them.
            have=$(sql "select count(*) from adlist where address='$url';")
            if [ "$have" = "0" ]; then
              echo "pihole-adlists: adding $url"
              sql "insert into adlist (address,enabled,comment) values ('$url',1,'declared in catalog/pihole.yaml');"
              CHANGED=1
            else
              en=$(sql "select enabled from adlist where address='$url';")
              if [ "$en" != "1" ]; then
                echo "pihole-adlists: re-enabling $url"
                sql "update adlist set enabled=1 where address='$url';"
                CHANGED=1
              fi
            fi
          done < /tmp/declared.txt

          # 2) Anything enabled in the DB but NOT declared gets disabled (never deleted - see above).
          sql "select address from adlist where enabled=1;" | while IFS= read -r addr; do
            [ -n "$addr" ] || continue
            if ! grep -qxF "$addr" /tmp/declared.txt; then
              echo "pihole-adlists: disabling undeclared $addr"
              sql "update adlist set enabled=0 where address='$addr';"
              echo changed >> /tmp/adlist-changed
            fi
          done
          [ -s /tmp/adlist-changed ] && CHANGED=1
          rm -f /tmp/adlist-changed /tmp/declared.txt

          # 3) Rebuild gravity ONLY if the declared set actually moved.
          if [ "$CHANGED" = "1" ]; then
            echo "pihole-adlists: declared set changed - running gravity update"
            pihole -g || echo "pihole-adlists: gravity update failed; existing gravity.db retained" >&2
          else
            echo "pihole-adlists: already matches the declared set - nothing to do"
          fi
          exit 0
update:
  channel: stable             # R-UPD-01
  range: ">=2024.0.0"
  cadence: "0 4 * * *"
  autoRollbackOnFailedHealth: true   # R-UPD-03
observability:
  metrics: true
  slis:
    - up                      # R-OBS-01
backup:
  policy: daily-offcluster    # PiHole config is DR-critical (docs/04 line 118; R-STATE-03)