From 82a9747fb95fe610c82f8effb79cad10bf77d259 Mon Sep 17 00:00:00 2001
From: MichaelFisher1997 <contact@michaelfisher.tech>
Date: Sun, 3 May 2026 20:15:12 +0000
Subject: [PATCH] fix: fail fast during rancher secret wait

---
 .gitea/workflows/deploy.yml | 75 +++++++++++++++++++++++++++++++------
 1 file changed, 64 insertions(+), 11 deletions(-)

diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml
index 3254555..df4559b 100644
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -594,6 +594,7 @@ jobs:
         env:
           KUBECONFIG: outputs/kubeconfig
           RANCHER_BOOTSTRAP_PASSWORD: ${{ secrets.RANCHER_BOOTSTRAP_PASSWORD }}
+          DOPPLER_HETZNERTERRA_SERVICE_TOKEN: ${{ secrets.DOPPLER_HETZNERTERRA_SERVICE_TOKEN }}
         run: |
           set -euo pipefail
           wait_for_resource() {
@@ -809,21 +810,73 @@ jobs:
             local timeout_seconds="$1"
             local deadline
             local force_sync
+            local api_failures=0
             deadline=$(($(date +%s) + timeout_seconds))
 
-            while [ "$(date +%s)" -lt "${deadline}" ]; do
-              force_sync="$(date +%s)"
-              kubectl -n flux-system annotate externalsecret/rancher-bootstrap-password external-secrets.io/force-sync="${force_sync}" --overwrite || true
-              kubectl -n cattle-system annotate externalsecret/rancher-bootstrap-password external-secrets.io/force-sync="${force_sync}" --overwrite || true
-
-              if [ -n "${RANCHER_BOOTSTRAP_PASSWORD}" ]; then
-                for namespace in flux-system cattle-system; do
-                  kubectl -n "${namespace}" create secret generic rancher-bootstrap-password \
-                    --from-literal=bootstrapPassword="${RANCHER_BOOTSTRAP_PASSWORD}" \
-                    --dry-run=client -o yaml | kubectl apply -f -
-                done
+            if [ -z "${RANCHER_BOOTSTRAP_PASSWORD}" ]; then
+              if [ -z "${DOPPLER_HETZNERTERRA_SERVICE_TOKEN}" ]; then
+                echo "RANCHER_BOOTSTRAP_PASSWORD and DOPPLER_HETZNERTERRA_SERVICE_TOKEN are both empty; cannot seed Rancher bootstrap secret" >&2
+                exit 1
               fi
 
+              RANCHER_BOOTSTRAP_PASSWORD="$(python3 - <<'PY'
+          import json
+          import os
+          import sys
+          import urllib.error
+          import urllib.request
+
+          token = os.environ.get("DOPPLER_HETZNERTERRA_SERVICE_TOKEN", "")
+          request = urllib.request.Request(
+              "https://api.doppler.com/v3/configs/config/secret?name=RANCHER_BOOTSTRAP_PASSWORD",
+              headers={"Authorization": f"Bearer {token}"},
+          )
+          try:
+              with urllib.request.urlopen(request, timeout=30) as response:
+                  payload = json.load(response)
+          except Exception as exc:
+              print(f"failed to load RANCHER_BOOTSTRAP_PASSWORD from Doppler: {exc}", file=sys.stderr)
+              sys.exit(1)
+
+          value = payload.get("value")
+          if isinstance(value, dict):
+              secret = value.get("raw") or value.get("computed") or value.get("value")
+          else:
+              secret = value
+
+          if not secret:
+              print("Doppler returned an empty RANCHER_BOOTSTRAP_PASSWORD", file=sys.stderr)
+              sys.exit(1)
+
+          print(secret)
+          PY
+              )"
+              export RANCHER_BOOTSTRAP_PASSWORD
+            fi
+
+            while [ "$(date +%s)" -lt "${deadline}" ]; do
+              if ! kubectl get --raw=/readyz --request-timeout=10s >/dev/null 2>&1; then
+                api_failures=$((api_failures + 1))
+                echo "Kubernetes API is not ready while waiting for Rancher bootstrap secrets (${api_failures}/6)" >&2
+                if [ "${api_failures}" -ge 6 ]; then
+                  kubectl -n kube-system get pods -o wide || true
+                  exit 1
+                fi
+                sleep 10
+                continue
+              fi
+              api_failures=0
+
+              force_sync="$(date +%s)"
+              kubectl -n flux-system annotate externalsecret/rancher-bootstrap-password external-secrets.io/force-sync="${force_sync}" --overwrite >/dev/null 2>&1 || true
+              kubectl -n cattle-system annotate externalsecret/rancher-bootstrap-password external-secrets.io/force-sync="${force_sync}" --overwrite >/dev/null 2>&1 || true
+
+              for namespace in flux-system cattle-system; do
+                kubectl -n "${namespace}" create secret generic rancher-bootstrap-password \
+                  --from-literal=bootstrapPassword="${RANCHER_BOOTSTRAP_PASSWORD}" \
+                  --dry-run=client -o yaml | kubectl apply -f - >/dev/null
+              done
+
               if kubectl -n flux-system get secret/rancher-bootstrap-password >/dev/null 2>&1 \
                 && kubectl -n cattle-system get secret/rancher-bootstrap-password >/dev/null 2>&1; then
                 return 0