fix: retry Rancher chart pulls during waits

2026-04-25 22:03:09 +00:00
parent d0be48b65c
commit c7a375758f
1 changed files with 28 additions and 0 deletions
@@ -553,6 +553,31 @@ jobs:
              --overwrite
          }

+          wait_for_helmchart_ready() {
+            local chart_name="$1"
+            local release_name="$2"
+            local timeout="$3"
+            local attempts="$4"
+            local reconcile_at
+
+            wait_for_resource flux-system "helmchart.source.toolkit.fluxcd.io/${chart_name}" 600
+            for attempt in $(seq 1 "${attempts}"); do
+              reconcile_at="$(date +%s)"
+              kubectl -n flux-system annotate "helmchart.source.toolkit.fluxcd.io/${chart_name}" reconcile.fluxcd.io/requestedAt="${reconcile_at}" --overwrite
+              kubectl -n flux-system annotate "helmrelease/${release_name}" reconcile.fluxcd.io/requestedAt="${reconcile_at}" --overwrite
+
+              if kubectl -n flux-system wait --for=condition=Ready "helmchart.source.toolkit.fluxcd.io/${chart_name}" --timeout="${timeout}"; then
+                return 0
+              fi
+
+              echo "HelmChart ${chart_name} did not become Ready after ${timeout}; forcing retry (${attempt}/${attempts})" >&2
+            done
+
+            kubectl -n flux-system describe "helmchart.source.toolkit.fluxcd.io/${chart_name}" || true
+            kubectl -n flux-system describe "helmrelease/${release_name}" || true
+            exit 1
+          }
+
          wait_for_helmrelease_ready() {
            local release_name="$1"
            local target_namespace="$2"
@@ -591,6 +616,7 @@ jobs:
          kubectl -n flux-system annotate kustomization/addon-rancher reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite
          wait_for_resource flux-system helmrelease.helm.toolkit.fluxcd.io/rancher 600
          reconcile_helmrelease rancher
+          wait_for_helmchart_ready flux-system-rancher rancher 900s 6
          wait_for_helmrelease_ready rancher cattle-system 900
          wait_for_resource "" namespace/cattle-system 600
          wait_for_resource cattle-system deployment/cattle-system-rancher 600
@@ -609,6 +635,8 @@ jobs:
          wait_for_resource flux-system helmrelease.helm.toolkit.fluxcd.io/rancher-backup 600
          reconcile_helmrelease rancher-backup-crd
          reconcile_helmrelease rancher-backup
+          wait_for_helmchart_ready flux-system-rancher-backup-crd rancher-backup-crd 600s 6
+          wait_for_helmchart_ready flux-system-rancher-backup rancher-backup 600s 6
          wait_for_helmrelease_ready rancher-backup-crd cattle-resources-system 600
          wait_for_helmrelease_ready rancher-backup cattle-resources-system 600
          wait_for_resource "" namespace/cattle-resources-system 600