feat: Add CloudNativePG with B2 backups for persistent Rancher database

- Add Local Path Provisioner for storage
- Add CloudNativePG operator (v1.27.0) via Flux
- Create PostgreSQL cluster with B2 (Backblaze) auto-backup/restore
- Update Rancher to use external PostgreSQL via CATTLE_DB_CATTLE_* env vars
- Add weekly pg_dump CronJob to B2 (Sundays 2AM)
- Add pre-destroy backup hook to destroy workflow
- Add B2 credentials to Doppler (B2_ACCOUNT_ID, B2_APPLICATION_KEY)
- Generate RANCHER_DB_PASSWORD in Doppler

Backup location: HetznerTerra/rancher-backups/
Retention: 14 backups

infrastructure/addons/cnpg/b2-credentials-externalsecret.yaml

@@ -0,0 +1,25 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: b2-credentials
+  namespace: cnpg-cluster
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: doppler-hetznerterra
+    kind: ClusterSecretStore
+  target:
+    name: b2-credentials
+    creationPolicy: Owner
+    template:
+      type: Opaque
+      data:
+        B2_ACCOUNT_ID: "{{ .B2_ACCOUNT_ID }}"
+        B2_APPLICATION_KEY: "{{ .B2_APPLICATION_KEY }}"
+  data:
+    - secretKey: B2_ACCOUNT_ID
+      remoteRef:
+        key: B2_ACCOUNT_ID
+    - secretKey: B2_APPLICATION_KEY
+      remoteRef:
+        key: B2_APPLICATION_KEY

infrastructure/addons/cnpg/cnpg-cluster-rw-svc.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: cnpg-cluster-rw
+  namespace: cnpg-cluster
+  labels:
+    app.kubernetes.io/name: rancher-db
+    cnpg.io/cluster: rancher-db
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - port: 5432
+      targetPort: 5432
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: postgresql
+    cnpg.io/cluster: rancher-db
+    role: primary

infrastructure/addons/cnpg/helmrelease-cnpg.yaml

@@ -0,0 +1,27 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cnpg
+  namespace: flux-system
+spec:
+  interval: 10m
+  targetNamespace: cnpg-system
+  chart:
+    spec:
+      chart: cloudnative-pg
+      version: 1.27.0
+      sourceRef:
+        kind: HelmRepository
+        name: cnpg
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  values:
+    image:
+      repository: ghcr.io/cloudnative-pg/postgresql
+    clusterImage: ghcr.io/cloudnative-pg/postgresql:17.4

infrastructure/addons/cnpg/helmrepository-cnpg.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: cnpg
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://cloudnative-pg.github.io/charts

infrastructure/addons/cnpg/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helmrepository-cnpg.yaml
+  - helmrelease-cnpg.yaml
+  - namespace.yaml
+  - b2-credentials-externalsecret.yaml
+  - rancher-db-password-externalsecret.yaml
+  - postgres-cluster.yaml
+  - cnpg-cluster-rw-svc.yaml
+  - pgdump-cronjob.yaml

infrastructure/addons/cnpg/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cnpg-cluster

infrastructure/addons/cnpg/pgdump-cronjob.yaml

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: pgdump-rancher
+  namespace: cnpg-cluster
+spec:
+  schedule: "0 2 * * 0"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 4
+  failedJobsHistoryLimit: 4
+  jobTemplate:
+    spec:
+      backoffLimit: 3
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: pgdump
+              image: ghcr.io/cloudnative-pg/pgbackrest:latest
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -e
+                  export AWS_ACCESS_KEY_ID=$(cat /etc/b2/credentials/B2_ACCOUNT_ID)
+                  export AWS_SECRET_ACCESS_KEY=$(cat /etc/b2/credentials/B2_APPLICATION_KEY)
+                  export AWS_ENDPOINT=https://s3.us-east-005.backblazeb2.com
+
+                  BACKUP_FILE="rancher-backup-$(date +%Y%m%d-%H%M%S).sql.gz"
+
+                  pg_dump -h cnpg-cluster-rw.cnpg-cluster.svc -U postgres -d postgres --no-owner --clean | gzip | \
+                    aws s3 cp - s3://HetznerTerra/rancher-backups/$BACKUP_FILE
+
+                  echo "Backup completed: $BACKUP_FILE"
+              env:
+                - name: PGPASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: rancher-db-password
+                      key: password
+              volumeMounts:
+                - name: b2-credentials
+                  mountPath: /etc/b2/credentials
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+                limits:
+                  cpu: 500m
+                  memory: 512Mi
+          volumes:
+            - name: b2-credentials
+              secret:
+                secretName: b2-credentials
+          nodeSelector:
+            kubernetes.io/hostname: k8s-cluster-cp-1
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule

infrastructure/addons/cnpg/postgres-cluster.yaml

@@ -0,0 +1,70 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: rancher-db
+  namespace: cnpg-cluster
+spec:
+  description: "Rancher external database cluster"
+  imageName: ghcr.io/cloudnative-pg/postgresql:17.4
+  imagePullPolicy: IfNotPresent
+
+  instances: 1
+  primaryUpdateStrategy: unsupervised
+
+  storage:
+    storageClass: local-path
+    size: 50Gi
+    resizeStorageStorageClassName: local-path
+
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: 1000m
+      memory: 2Gi
+
+  bootstrap:
+    recovery:
+      externalClusters:
+        - name: b2-backup
+          s3Compatible:
+            bucket: HetznerTerra
+            region: us-east-005
+            endpoint: s3.us-east-005.backblazeb2.com
+            prefix: rancher-backups/
+            credentials:
+              name: b2-credentials
+              accessKey: B2_ACCOUNT_ID
+              secretKey: B2_APPLICATION_KEY
+
+  backup:
+    b2:
+      bucket: HetznerTerra
+      region: us-east-005
+      endpoint: s3.us-east-005.backblazeb2.com
+      prefix: rancher-backups/
+      credentials:
+        name: b2-credentials
+        accessKey: B2_ACCOUNT_ID
+        secretKey: B2_APPLICATION_KEY
+    retentionPolicy: keep14
+
+  serviceAccountTemplate:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rancher-db
+
+  superuserSecret:
+    name: rancher-db-password
+
+  monitoring:
+    enablePodMonitor: true
+
+  affinity:
+    nodeSelector:
+      kubernetes.io/hostname: k8s-cluster-cp-1
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

infrastructure/addons/cnpg/rancher-db-password-externalsecret.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: rancher-db-password
+  namespace: cnpg-cluster
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: doppler-hetznerterra
+    kind: ClusterSecretStore
+  target:
+    name: rancher-db-password
+    creationPolicy: Owner
+    template:
+      type: Opaque
+      data:
+        password: "{{ .RANCHER_DB_PASSWORD }}"
+  data:
+    - secretKey: RANCHER_DB_PASSWORD
+      remoteRef:
+        key: RANCHER_DB_PASSWORD