From f2c506b350d4f6dbfafc6b5f760db1749ae88111 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sun, 29 Mar 2026 21:53:16 +0000 Subject: [PATCH] refactor: Replace CNPG external DB with rancher-backup operator Rancher 2.x uses embedded etcd, not an external PostgreSQL database. The CATTLE_DB_CATTLE_* env vars are Rancher v1 only and were ignored. - Remove all CNPG (CloudNativePG) cluster, operator, and related configs - Remove external DB env vars from Rancher HelmRelease - Remove rancher-db-password ExternalSecret - Add rancher-backup operator HelmRelease (v106.0.2+up8.1.0) - Add B2 credentials ExternalSecret for backup storage - Add recurring Backup CR (daily at 03:00, 7 day retention) - Add commented-out Restore CR for rebuild recovery - Update Flux dependency graph accordingly --- .../addons/cnpg-operator/kustomization.yaml | 5 -- .../addons/cnpg/cnpg-cluster-rw-svc.yaml | 19 ----- infrastructure/addons/cnpg/kustomization.yaml | 11 --- .../addons/cnpg/postgres-cluster.yaml | 79 ------------------- .../rancher-db-password-externalsecret.yaml | 21 ----- .../addons/cnpg/role-b2-reader.yaml | 10 --- .../addons/cnpg/rolebinding-b2-reader.yaml | 13 --- .../addons/cnpg/scheduled-backup-rancher.yaml | 11 --- .../addons/kustomization-cnpg-operator.yaml | 15 ---- ...yaml => kustomization-rancher-backup.yaml} | 11 +-- .../addons/kustomization-rancher.yaml | 1 - infrastructure/addons/kustomization.yaml | 3 +- .../b2-credentials-externalsecret.yaml | 12 +-- .../rancher-backup/backup-recurring.yaml | 17 ++++ .../helmrelease-rancher-backup.yaml} | 14 ++-- .../helmrepository-rancher-backup.yaml} | 4 +- .../addons/rancher-backup/kustomization.yaml | 9 +++ .../{cnpg => rancher-backup}/namespace.yaml | 2 +- .../rancher-backup/restore-from-b2.yaml | 19 +++++ .../addons/rancher/helmrelease-rancher.yaml | 13 --- .../addons/rancher/kustomization.yaml | 1 - .../rancher-db-password-externalsecret.yaml | 21 ----- 22 files changed, 66 insertions(+), 245 deletions(-) delete mode 100644 infrastructure/addons/cnpg-operator/kustomization.yaml delete mode 100644 infrastructure/addons/cnpg/cnpg-cluster-rw-svc.yaml delete mode 100644 infrastructure/addons/cnpg/kustomization.yaml delete mode 100644 infrastructure/addons/cnpg/postgres-cluster.yaml delete mode 100644 infrastructure/addons/cnpg/rancher-db-password-externalsecret.yaml delete mode 100644 infrastructure/addons/cnpg/role-b2-reader.yaml delete mode 100644 infrastructure/addons/cnpg/rolebinding-b2-reader.yaml delete mode 100644 infrastructure/addons/cnpg/scheduled-backup-rancher.yaml delete mode 100644 infrastructure/addons/kustomization-cnpg-operator.yaml rename infrastructure/addons/{kustomization-cnpg.yaml => kustomization-rancher-backup.yaml} (58%) rename infrastructure/addons/{cnpg => rancher-backup}/b2-credentials-externalsecret.yaml (62%) create mode 100644 infrastructure/addons/rancher-backup/backup-recurring.yaml rename infrastructure/addons/{cnpg-operator/helmrelease-cnpg.yaml => rancher-backup/helmrelease-rancher-backup.yaml} (59%) rename infrastructure/addons/{cnpg-operator/helmrepository-cnpg.yaml => rancher-backup/helmrepository-rancher-backup.yaml} (66%) create mode 100644 infrastructure/addons/rancher-backup/kustomization.yaml rename infrastructure/addons/{cnpg => rancher-backup}/namespace.yaml (56%) create mode 100644 infrastructure/addons/rancher-backup/restore-from-b2.yaml delete mode 100644 infrastructure/addons/rancher/rancher-db-password-externalsecret.yaml diff --git a/infrastructure/addons/cnpg-operator/kustomization.yaml b/infrastructure/addons/cnpg-operator/kustomization.yaml deleted file mode 100644 index f2a949e..0000000 --- a/infrastructure/addons/cnpg-operator/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - helmrepository-cnpg.yaml - - helmrelease-cnpg.yaml \ No newline at end of file diff --git a/infrastructure/addons/cnpg/cnpg-cluster-rw-svc.yaml b/infrastructure/addons/cnpg/cnpg-cluster-rw-svc.yaml deleted file mode 100644 index 12c011c..0000000 --- a/infrastructure/addons/cnpg/cnpg-cluster-rw-svc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: cnpg-cluster-rw - namespace: cnpg-cluster - labels: - app.kubernetes.io/name: rancher-db - cnpg.io/cluster: rancher-db -spec: - type: ClusterIP - clusterIP: None - ports: - - port: 5432 - targetPort: 5432 - protocol: TCP - selector: - app.kubernetes.io/name: postgresql - cnpg.io/cluster: rancher-db - role: primary \ No newline at end of file diff --git a/infrastructure/addons/cnpg/kustomization.yaml b/infrastructure/addons/cnpg/kustomization.yaml deleted file mode 100644 index 1fd6e97..0000000 --- a/infrastructure/addons/cnpg/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - b2-credentials-externalsecret.yaml - - rancher-db-password-externalsecret.yaml - - postgres-cluster.yaml - - cnpg-cluster-rw-svc.yaml - - role-b2-reader.yaml - - rolebinding-b2-reader.yaml - - scheduled-backup-rancher.yaml diff --git a/infrastructure/addons/cnpg/postgres-cluster.yaml b/infrastructure/addons/cnpg/postgres-cluster.yaml deleted file mode 100644 index c06d28f..0000000 --- a/infrastructure/addons/cnpg/postgres-cluster.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: rancher-db - namespace: cnpg-cluster - annotations: - cnpg.io/skipEmptyWalArchiveCheck: "enabled" -spec: - description: "Rancher external database cluster" - imageName: ghcr.io/cloudnative-pg/postgresql:17.4 - imagePullPolicy: IfNotPresent - - instances: 1 - primaryUpdateStrategy: unsupervised - - storage: - storageClass: local-path - size: 50Gi - - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 1000m - memory: 2Gi - - serviceAccountTemplate: - metadata: - labels: - app.kubernetes.io/name: rancher-db - - superuserSecret: - name: rancher-db-password - - bootstrap: - recovery: - source: b2-backup - - externalClusters: - - name: b2-backup - barmanObjectStore: - destinationPath: "s3://HetznerTerra/rancher-backups/" - endpointURL: "https://s3.us-east-005.backblazeb2.com" - serverName: rancher-db - s3Credentials: - accessKeyId: - name: b2-credentials - key: B2_ACCOUNT_ID - secretAccessKey: - name: b2-credentials - key: B2_APPLICATION_KEY - - monitoring: - enablePodMonitor: false - - affinity: - nodeSelector: - kubernetes.io/hostname: k8s-cluster-cp-1 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - backup: - barmanObjectStore: - destinationPath: "s3://HetznerTerra/rancher-backups/" - endpointURL: "https://s3.us-east-005.backblazeb2.com" - s3Credentials: - accessKeyId: - name: b2-credentials - key: B2_ACCOUNT_ID - secretAccessKey: - name: b2-credentials - key: B2_APPLICATION_KEY - wal: - compression: gzip - data: - compression: gzip diff --git a/infrastructure/addons/cnpg/rancher-db-password-externalsecret.yaml b/infrastructure/addons/cnpg/rancher-db-password-externalsecret.yaml deleted file mode 100644 index eaf2b24..0000000 --- a/infrastructure/addons/cnpg/rancher-db-password-externalsecret.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: rancher-db-password - namespace: cnpg-cluster -spec: - refreshInterval: 1h - secretStoreRef: - name: doppler-hetznerterra - kind: ClusterSecretStore - target: - name: rancher-db-password - creationPolicy: Owner - template: - type: Opaque - data: - password: "{{ .RANCHER_DB_PASSWORD }}" - data: - - secretKey: RANCHER_DB_PASSWORD - remoteRef: - key: RANCHER_DB_PASSWORD \ No newline at end of file diff --git a/infrastructure/addons/cnpg/role-b2-reader.yaml b/infrastructure/addons/cnpg/role-b2-reader.yaml deleted file mode 100644 index 05ad598..0000000 --- a/infrastructure/addons/cnpg/role-b2-reader.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: rancher-db-b2-reader - namespace: cnpg-cluster -rules: - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["b2-credentials"] - verbs: ["get", "list"] \ No newline at end of file diff --git a/infrastructure/addons/cnpg/rolebinding-b2-reader.yaml b/infrastructure/addons/cnpg/rolebinding-b2-reader.yaml deleted file mode 100644 index d62901c..0000000 --- a/infrastructure/addons/cnpg/rolebinding-b2-reader.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rancher-db-b2-reader - namespace: cnpg-cluster -subjects: - - kind: ServiceAccount - name: rancher-db - namespace: cnpg-cluster -roleRef: - kind: Role - name: rancher-db-b2-reader - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/infrastructure/addons/cnpg/scheduled-backup-rancher.yaml b/infrastructure/addons/cnpg/scheduled-backup-rancher.yaml deleted file mode 100644 index 7deffa3..0000000 --- a/infrastructure/addons/cnpg/scheduled-backup-rancher.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: ScheduledBackup -metadata: - name: rancher-db-weekly - namespace: cnpg-cluster -spec: - schedule: "0 0 2 * * 0" - backupOwnerReference: self - cluster: - name: rancher-db - target: primary diff --git a/infrastructure/addons/kustomization-cnpg-operator.yaml b/infrastructure/addons/kustomization-cnpg-operator.yaml deleted file mode 100644 index 4ed6cb7..0000000 --- a/infrastructure/addons/kustomization-cnpg-operator.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: addon-cnpg-operator - namespace: flux-system -spec: - interval: 10m - prune: true - sourceRef: - kind: GitRepository - name: platform - path: ./infrastructure/addons/cnpg-operator - wait: true - timeout: 10m - suspend: false \ No newline at end of file diff --git a/infrastructure/addons/kustomization-cnpg.yaml b/infrastructure/addons/kustomization-rancher-backup.yaml similarity index 58% rename from infrastructure/addons/kustomization-cnpg.yaml rename to infrastructure/addons/kustomization-rancher-backup.yaml index 4eff6ee..7dc4a5c 100644 --- a/infrastructure/addons/kustomization-cnpg.yaml +++ b/infrastructure/addons/kustomization-rancher-backup.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: addon-cnpg + name: addon-rancher-backup namespace: flux-system spec: interval: 10m @@ -9,15 +9,10 @@ spec: sourceRef: kind: GitRepository name: platform - path: ./infrastructure/addons/cnpg + path: ./infrastructure/addons/rancher-backup wait: true timeout: 10m suspend: false dependsOn: - - name: addon-cnpg-operator - name: addon-external-secrets - healthChecks: - - apiVersion: v1 - kind: Secret - name: b2-credentials - namespace: cnpg-cluster \ No newline at end of file + - name: addon-rancher diff --git a/infrastructure/addons/kustomization-rancher.yaml b/infrastructure/addons/kustomization-rancher.yaml index 46ba79c..e27a983 100644 --- a/infrastructure/addons/kustomization-rancher.yaml +++ b/infrastructure/addons/kustomization-rancher.yaml @@ -17,4 +17,3 @@ spec: - name: addon-tailscale-operator - name: addon-tailscale-proxyclass - name: addon-external-secrets - - name: addon-cnpg diff --git a/infrastructure/addons/kustomization.yaml b/infrastructure/addons/kustomization.yaml index 9944d0e..c703710 100644 --- a/infrastructure/addons/kustomization.yaml +++ b/infrastructure/addons/kustomization.yaml @@ -10,7 +10,6 @@ resources: - kustomization-flux-ui.yaml - kustomization-observability.yaml - kustomization-observability-content.yaml - - kustomization-cnpg-operator.yaml - - kustomization-cnpg.yaml - kustomization-rancher.yaml - kustomization-rancher-config.yaml + - kustomization-rancher-backup.yaml diff --git a/infrastructure/addons/cnpg/b2-credentials-externalsecret.yaml b/infrastructure/addons/rancher-backup/b2-credentials-externalsecret.yaml similarity index 62% rename from infrastructure/addons/cnpg/b2-credentials-externalsecret.yaml rename to infrastructure/addons/rancher-backup/b2-credentials-externalsecret.yaml index 93b1928..5acbcbd 100644 --- a/infrastructure/addons/cnpg/b2-credentials-externalsecret.yaml +++ b/infrastructure/addons/rancher-backup/b2-credentials-externalsecret.yaml @@ -1,25 +1,25 @@ apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: b2-credentials - namespace: cnpg-cluster + name: rancher-b2-creds + namespace: cattle-resources-system spec: refreshInterval: 1h secretStoreRef: name: doppler-hetznerterra kind: ClusterSecretStore target: - name: b2-credentials + name: rancher-b2-creds creationPolicy: Owner template: type: Opaque data: - B2_ACCOUNT_ID: "{{ .B2_ACCOUNT_ID }}" - B2_APPLICATION_KEY: "{{ .B2_APPLICATION_KEY }}" + aws_access_key_id: "{{ .B2_ACCOUNT_ID }}" + aws_secret_access_key: "{{ .B2_APPLICATION_KEY }}" data: - secretKey: B2_ACCOUNT_ID remoteRef: key: B2_ACCOUNT_ID - secretKey: B2_APPLICATION_KEY remoteRef: - key: B2_APPLICATION_KEY \ No newline at end of file + key: B2_APPLICATION_KEY diff --git a/infrastructure/addons/rancher-backup/backup-recurring.yaml b/infrastructure/addons/rancher-backup/backup-recurring.yaml new file mode 100644 index 0000000..5b15a02 --- /dev/null +++ b/infrastructure/addons/rancher-backup/backup-recurring.yaml @@ -0,0 +1,17 @@ +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: rancher-b2-recurring + namespace: cattle-resources-system +spec: + resourceSetName: rancher-resource-set-full + storageLocation: + s3: + credentialSecretName: rancher-b2-creds + credentialSecretNamespace: cattle-resources-system + bucketName: HetznerTerra + folder: rancher-backups + endpoint: s3.us-east-005.backblazeb2.com + region: us-east-005 + schedule: "0 3 * * *" + retentionCount: 7 diff --git a/infrastructure/addons/cnpg-operator/helmrelease-cnpg.yaml b/infrastructure/addons/rancher-backup/helmrelease-rancher-backup.yaml similarity index 59% rename from infrastructure/addons/cnpg-operator/helmrelease-cnpg.yaml rename to infrastructure/addons/rancher-backup/helmrelease-rancher-backup.yaml index 200573a..02db017 100644 --- a/infrastructure/addons/cnpg-operator/helmrelease-cnpg.yaml +++ b/infrastructure/addons/rancher-backup/helmrelease-rancher-backup.yaml @@ -1,18 +1,18 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: cnpg + name: rancher-backup namespace: flux-system spec: interval: 10m - targetNamespace: cnpg-system + targetNamespace: cattle-resources-system chart: spec: - chart: cloudnative-pg - version: 0.27.1 + chart: rancher-backup + version: "106.0.2+up8.1.0" sourceRef: kind: HelmRepository - name: cnpg + name: rancher-charts namespace: flux-system install: createNamespace: true @@ -23,4 +23,6 @@ spec: retries: 3 values: image: - repository: ghcr.io/cloudnative-pg/cloudnative-pg \ No newline at end of file + repository: rancher/backup-restore-operator + s3: + enabled: true diff --git a/infrastructure/addons/cnpg-operator/helmrepository-cnpg.yaml b/infrastructure/addons/rancher-backup/helmrepository-rancher-backup.yaml similarity index 66% rename from infrastructure/addons/cnpg-operator/helmrepository-cnpg.yaml rename to infrastructure/addons/rancher-backup/helmrepository-rancher-backup.yaml index 88705dc..503f35c 100644 --- a/infrastructure/addons/cnpg-operator/helmrepository-cnpg.yaml +++ b/infrastructure/addons/rancher-backup/helmrepository-rancher-backup.yaml @@ -1,8 +1,8 @@ apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: - name: cnpg + name: rancher-charts namespace: flux-system spec: interval: 1h - url: https://cloudnative-pg.github.io/charts \ No newline at end of file + url: https://charts.rancher.io diff --git a/infrastructure/addons/rancher-backup/kustomization.yaml b/infrastructure/addons/rancher-backup/kustomization.yaml new file mode 100644 index 0000000..16c6db1 --- /dev/null +++ b/infrastructure/addons/rancher-backup/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helmrepository-rancher-backup.yaml + - helmrelease-rancher-backup.yaml + - b2-credentials-externalsecret.yaml + - backup-recurring.yaml + - restore-from-b2.yaml diff --git a/infrastructure/addons/cnpg/namespace.yaml b/infrastructure/addons/rancher-backup/namespace.yaml similarity index 56% rename from infrastructure/addons/cnpg/namespace.yaml rename to infrastructure/addons/rancher-backup/namespace.yaml index 385a070..c6b75df 100644 --- a/infrastructure/addons/cnpg/namespace.yaml +++ b/infrastructure/addons/rancher-backup/namespace.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: cnpg-cluster \ No newline at end of file + name: cattle-resources-system diff --git a/infrastructure/addons/rancher-backup/restore-from-b2.yaml b/infrastructure/addons/rancher-backup/restore-from-b2.yaml new file mode 100644 index 0000000..049393a --- /dev/null +++ b/infrastructure/addons/rancher-backup/restore-from-b2.yaml @@ -0,0 +1,19 @@ +# Uncomment and set backupFilename to restore from a specific backup on rebuild. +# Find the latest backup filename in B2: rancher-backups/ folder. +# After restore succeeds, Rancher will have all users/settings from the backup. +# +# apiVersion: resources.cattle.io/v1 +# kind: Restore +# metadata: +# name: restore-from-b2 +# namespace: cattle-resources-system +# spec: +# backupFilename: +# storageLocation: +# s3: +# credentialSecretName: rancher-b2-creds +# credentialSecretNamespace: cattle-resources-system +# bucketName: HetznerTerra +# folder: rancher-backups +# endpoint: s3.us-east-005.backblazeb2.com +# region: us-east-005 diff --git a/infrastructure/addons/rancher/helmrelease-rancher.yaml b/infrastructure/addons/rancher/helmrelease-rancher.yaml index 53282f6..4627f2a 100644 --- a/infrastructure/addons/rancher/helmrelease-rancher.yaml +++ b/infrastructure/addons/rancher/helmrelease-rancher.yaml @@ -26,19 +26,6 @@ spec: tls: external replicas: 1 extraEnv: - - name: CATTLE_DB_CATTLE_HOST - value: cnpg-cluster-rw.cnpg-cluster.svc - - name: CATTLE_DB_CATTLE_PORT - value: "5432" - - name: CATTLE_DB_CATTLE_DATABASE - value: postgres - - name: CATTLE_DB_CATTLE_USERNAME - value: postgres - - name: CATTLE_DB_CATTLE_PASSWORD - valueFrom: - secretKeyRef: - name: rancher-db-password - key: password - name: CATTLE_PROMETHEUS_METRICS value: "true" resources: diff --git a/infrastructure/addons/rancher/kustomization.yaml b/infrastructure/addons/rancher/kustomization.yaml index 11901e5..8c03910 100644 --- a/infrastructure/addons/rancher/kustomization.yaml +++ b/infrastructure/addons/rancher/kustomization.yaml @@ -6,5 +6,4 @@ resources: - helmrelease-rancher.yaml - rancher-bootstrap-password-flux-externalsecret.yaml - rancher-bootstrap-password-externalsecret.yaml - - rancher-db-password-externalsecret.yaml - rancher-tailscale-service.yaml diff --git a/infrastructure/addons/rancher/rancher-db-password-externalsecret.yaml b/infrastructure/addons/rancher/rancher-db-password-externalsecret.yaml deleted file mode 100644 index 106037e..0000000 --- a/infrastructure/addons/rancher/rancher-db-password-externalsecret.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: rancher-db-password - namespace: cattle-system -spec: - refreshInterval: 1h - secretStoreRef: - name: doppler-hetznerterra - kind: ClusterSecretStore - target: - name: rancher-db-password - creationPolicy: Owner - template: - type: Opaque - data: - password: "{{ .RANCHER_DB_PASSWORD }}" - data: - - secretKey: RANCHER_DB_PASSWORD - remoteRef: - key: RANCHER_DB_PASSWORD