fix: decouple observability secret health gate

2026-05-02 00:05:55 +00:00
parent f885d8ab2e
commit 524147dac3
3 changed files with 48 additions and 9 deletions
@@ -884,6 +884,9 @@ jobs:
            kubectl -n flux-system describe kustomization/addon-observability-secrets || true
            kubectl -n flux-system describe kustomization/addon-observability || true
            kubectl -n flux-system describe kustomization/addon-observability-content || true
+            kubectl describe clustersecretstore/doppler-hetznerterra || true
+            kubectl -n observability describe externalsecret/grafana-admin || true
+            kubectl -n observability get secret/grafana-admin-credentials || true
            kubectl -n flux-system describe ocirepository/loki || true
            kubectl -n flux-system describe ocirepository/promtail || true
            kubectl -n flux-system describe helmrelease/kube-prometheus-stack || true
@@ -962,6 +965,26 @@ jobs:
            fi
          }

+          wait_for_grafana_secret() {
+            local timeout_seconds="$1"
+            local elapsed=0
+
+            while [ "${elapsed}" -lt "${timeout_seconds}" ]; do
+              if kubectl wait --for=condition=Ready clustersecretstore/doppler-hetznerterra --timeout=30s \
+                && kubectl -n observability wait --for=condition=Ready externalsecret/grafana-admin --timeout=30s \
+                && kubectl -n observability get secret/grafana-admin-credentials >/dev/null 2>&1; then
+                return 0
+              fi
+
+              sleep 15
+              elapsed=$((elapsed + 75))
+            done
+
+            echo "Timed out waiting for Grafana admin ExternalSecret to sync" >&2
+            observability_diagnostics
+            exit 1
+          }
+
          wait_for_ocirepository_ready_or_cached() {
            local repository="$1"
            local timeout="$2"
@@ -1018,6 +1041,7 @@ jobs:
          wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability-secrets 600
          reconcile_flux_resource kustomization/addon-observability-secrets 300
          wait_for_flux_ready kustomization/addon-observability-secrets 300s
+          wait_for_grafana_secret 900
          wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability 600
          reconcile_flux_resource kustomization/addon-observability 600
          wait_for_flux_ready kustomization/addon-observability 300s