diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 7c14394..fffc0f7 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -884,6 +884,9 @@ jobs: kubectl -n flux-system describe kustomization/addon-observability-secrets || true kubectl -n flux-system describe kustomization/addon-observability || true kubectl -n flux-system describe kustomization/addon-observability-content || true + kubectl describe clustersecretstore/doppler-hetznerterra || true + kubectl -n observability describe externalsecret/grafana-admin || true + kubectl -n observability get secret/grafana-admin-credentials || true kubectl -n flux-system describe ocirepository/loki || true kubectl -n flux-system describe ocirepository/promtail || true kubectl -n flux-system describe helmrelease/kube-prometheus-stack || true @@ -962,6 +965,26 @@ jobs: fi } + wait_for_grafana_secret() { + local timeout_seconds="$1" + local elapsed=0 + + while [ "${elapsed}" -lt "${timeout_seconds}" ]; do + if kubectl wait --for=condition=Ready clustersecretstore/doppler-hetznerterra --timeout=30s \ + && kubectl -n observability wait --for=condition=Ready externalsecret/grafana-admin --timeout=30s \ + && kubectl -n observability get secret/grafana-admin-credentials >/dev/null 2>&1; then + return 0 + fi + + sleep 15 + elapsed=$((elapsed + 75)) + done + + echo "Timed out waiting for Grafana admin ExternalSecret to sync" >&2 + observability_diagnostics + exit 1 + } + wait_for_ocirepository_ready_or_cached() { local repository="$1" local timeout="$2" @@ -1018,6 +1041,7 @@ jobs: wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability-secrets 600 reconcile_flux_resource kustomization/addon-observability-secrets 300 wait_for_flux_ready kustomization/addon-observability-secrets 300s + wait_for_grafana_secret 900 wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability 600 reconcile_flux_resource kustomization/addon-observability 600 wait_for_flux_ready kustomization/addon-observability 300s diff --git a/.gitea/workflows/observability.yml b/.gitea/workflows/observability.yml index 1621c0d..0feb587 100644 --- a/.gitea/workflows/observability.yml +++ b/.gitea/workflows/observability.yml @@ -98,6 +98,9 @@ jobs: kubectl -n flux-system describe kustomization/addon-observability-secrets || true kubectl -n flux-system describe kustomization/addon-observability || true kubectl -n flux-system describe kustomization/addon-observability-content || true + kubectl describe clustersecretstore/doppler-hetznerterra || true + kubectl -n observability describe externalsecret/grafana-admin || true + kubectl -n observability get secret/grafana-admin-credentials || true kubectl -n flux-system describe ocirepository/loki || true kubectl -n flux-system describe ocirepository/promtail || true kubectl -n flux-system describe helmrelease/kube-prometheus-stack || true @@ -176,6 +179,26 @@ jobs: fi } + wait_for_grafana_secret() { + local timeout_seconds="$1" + local elapsed=0 + + while [ "${elapsed}" -lt "${timeout_seconds}" ]; do + if kubectl wait --for=condition=Ready clustersecretstore/doppler-hetznerterra --timeout=30s \ + && kubectl -n observability wait --for=condition=Ready externalsecret/grafana-admin --timeout=30s \ + && kubectl -n observability get secret/grafana-admin-credentials >/dev/null 2>&1; then + return 0 + fi + + sleep 15 + elapsed=$((elapsed + 75)) + done + + echo "Timed out waiting for Grafana admin ExternalSecret to sync" >&2 + observability_diagnostics + exit 1 + } + wait_for_ocirepository_ready_or_cached() { local repository="$1" local timeout="$2" @@ -237,6 +260,7 @@ jobs: wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability-secrets 300 reconcile_flux_resource kustomization/addon-observability-secrets 300 wait_for_flux_ready kustomization/addon-observability-secrets 300s + wait_for_grafana_secret 900 wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-observability 300 reconcile_flux_resource kustomization/addon-observability 600 wait_for_flux_ready kustomization/addon-observability 300s diff --git a/infrastructure/addons/kustomization-observability-secrets.yaml b/infrastructure/addons/kustomization-observability-secrets.yaml index 760ff85..bf01e63 100644 --- a/infrastructure/addons/kustomization-observability-secrets.yaml +++ b/infrastructure/addons/kustomization-observability-secrets.yaml @@ -13,14 +13,5 @@ spec: dependsOn: - name: addon-external-secrets-store wait: false - healthChecks: - - apiVersion: external-secrets.io/v1 - kind: ExternalSecret - name: grafana-admin - namespace: observability - - apiVersion: v1 - kind: Secret - name: grafana-admin-credentials - namespace: observability timeout: 5m suspend: false