fix: decouple observability secret health gate

.gitea/workflows/deploy.yml

@@ -884,6 +884,9 @@ jobs:
             kubectl -n flux-system describe kustomization/addon-observability-secrets || true
             kubectl -n flux-system describe kustomization/addon-observability || true
             kubectl -n flux-system describe kustomization/addon-observability-content || true
+            kubectl describe clustersecretstore/doppler-hetznerterra || true
+            kubectl -n observability describe externalsecret/grafana-admin || true
+            kubectl -n observability get secret/grafana-admin-credentials || true
             kubectl -n flux-system describe ocirepository/loki || true
             kubectl -n flux-system describe ocirepository/promtail || true
             kubectl -n flux-system describe helmrelease/kube-prometheus-stack || true
@@ -962,6 +965,26 @@ jobs:
             fi
           }
 
+          wait_for_grafana_secret() {
+            local timeout_seconds="$1"
+            local elapsed=0
+
+            while [ "${elapsed}" -lt "${timeout_seconds}" ]; do
+              if kubectl wait --for=condition=Ready clustersecretstore/doppler-hetznerterra --timeout=30s \
+                && kubectl -n observability wait --for=condition=Ready externalsecret/grafana-admin --timeout=30s \
+                && kubectl -n observability get secret/grafana-admin-credentials >/dev/null 2>&1; then
+                return 0
+              fi
+
+              sleep 15
+              elapsed=$((elapsed + 75))
+            done
+
+            echo "Timed out waiting for Grafana admin ExternalSecret to sync" >&2
+            observability_diagnostics
+            exit 1
+          }
+
           wait_for_ocirepository_ready_or_cached() {
             local repository="$1"
             local timeout="$2"
@@ -1018,6 +1041,7 @@ jobs:
           wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability-secrets 600
           reconcile_flux_resource kustomization/addon-observability-secrets 300
           wait_for_flux_ready kustomization/addon-observability-secrets 300s
+          wait_for_grafana_secret 900
           wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability 600
           reconcile_flux_resource kustomization/addon-observability 600
           wait_for_flux_ready kustomization/addon-observability 300s

.gitea/workflows/observability.yml

@@ -98,6 +98,9 @@ jobs:
             kubectl -n flux-system describe kustomization/addon-observability-secrets || true
             kubectl -n flux-system describe kustomization/addon-observability || true
             kubectl -n flux-system describe kustomization/addon-observability-content || true
+            kubectl describe clustersecretstore/doppler-hetznerterra || true
+            kubectl -n observability describe externalsecret/grafana-admin || true
+            kubectl -n observability get secret/grafana-admin-credentials || true
             kubectl -n flux-system describe ocirepository/loki || true
             kubectl -n flux-system describe ocirepository/promtail || true
             kubectl -n flux-system describe helmrelease/kube-prometheus-stack || true
@@ -176,6 +179,26 @@ jobs:
             fi
           }
 
+          wait_for_grafana_secret() {
+            local timeout_seconds="$1"
+            local elapsed=0
+
+            while [ "${elapsed}" -lt "${timeout_seconds}" ]; do
+              if kubectl wait --for=condition=Ready clustersecretstore/doppler-hetznerterra --timeout=30s \
+                && kubectl -n observability wait --for=condition=Ready externalsecret/grafana-admin --timeout=30s \
+                && kubectl -n observability get secret/grafana-admin-credentials >/dev/null 2>&1; then
+                return 0
+              fi
+
+              sleep 15
+              elapsed=$((elapsed + 75))
+            done
+
+            echo "Timed out waiting for Grafana admin ExternalSecret to sync" >&2
+            observability_diagnostics
+            exit 1
+          }
+
           wait_for_ocirepository_ready_or_cached() {
             local repository="$1"
             local timeout="$2"
@@ -237,6 +260,7 @@ jobs:
           wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability-secrets 300
           reconcile_flux_resource kustomization/addon-observability-secrets 300
           wait_for_flux_ready kustomization/addon-observability-secrets 300s
+          wait_for_grafana_secret 900
           wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability 300
           reconcile_flux_resource kustomization/addon-observability 600
           wait_for_flux_ready kustomization/addon-observability 300s

infrastructure/addons/kustomization-observability-secrets.yaml

@@ -13,14 +13,5 @@ spec:
   dependsOn:
     - name: addon-external-secrets-store
   wait: false
-  healthChecks:
-    - apiVersion: external-secrets.io/v1
-      kind: ExternalSecret
-      name: grafana-admin
-      namespace: observability
-    - apiVersion: v1
-      kind: Secret
-      name: grafana-admin-credentials
-      namespace: observability
   timeout: 5m
   suspend: false