diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 95e41c6..a1e30ac 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -270,10 +270,56 @@ jobs: kubectl -n flux-system get kustomizations,helmrepositories,helmcharts,helmreleases || true kubectl -n flux-system describe kustomization addon-external-secrets || true kubectl -n flux-system describe helmrepository external-secrets || true + kubectl -n flux-system describe helmchart flux-system-external-secrets || true kubectl -n flux-system describe helmrelease external-secrets || true kubectl -n external-secrets get pods -o wide || true } + flux_helm_diagnostics() { + local repo_name="$1" + local chart_name="$2" + local release_name="$3" + local target_namespace="$4" + + kubectl -n flux-system get helmrepositories,helmcharts,helmreleases || true + kubectl -n flux-system describe helmrepository "${repo_name}" || true + kubectl -n flux-system describe helmchart "${chart_name}" || true + kubectl -n flux-system describe helmrelease "${release_name}" || true + kubectl -n "${target_namespace}" get pods -o wide || true + } + + wait_for_flux_helm_release() { + local repo_name="$1" + local chart_name="$2" + local release_name="$3" + local target_namespace="$4" + local repo_timeout="$5" + local chart_timeout="$6" + local release_timeout="$7" + local reconcile_at + + wait_for_resource flux-system "helmrepository.source.toolkit.fluxcd.io/${repo_name}" 600 + if ! kubectl -n flux-system wait --for=condition=Ready "helmrepository/${repo_name}" --timeout="${repo_timeout}"; then + flux_helm_diagnostics "${repo_name}" "${chart_name}" "${release_name}" "${target_namespace}" + exit 1 + fi + + wait_for_resource flux-system "helmchart.source.toolkit.fluxcd.io/${chart_name}" 600 + reconcile_at="$(date +%s)" + kubectl -n flux-system annotate "helmchart/${chart_name}" reconcile.fluxcd.io/requestedAt="${reconcile_at}" --overwrite + kubectl -n flux-system annotate "helmrelease/${release_name}" reconcile.fluxcd.io/requestedAt="${reconcile_at}" --overwrite + + if ! kubectl -n flux-system wait --for=condition=Ready "helmchart/${chart_name}" --timeout="${chart_timeout}"; then + flux_helm_diagnostics "${repo_name}" "${chart_name}" "${release_name}" "${target_namespace}" + exit 1 + fi + + if ! kubectl -n flux-system wait --for=condition=Ready "helmrelease/${release_name}" --timeout="${release_timeout}"; then + flux_helm_diagnostics "${repo_name}" "${chart_name}" "${release_name}" "${target_namespace}" + exit 1 + fi + } + kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - ssh-keyscan -p "${FLUX_GIT_PORT}" "${FLUX_GIT_HOST}" > /tmp/flux_known_hosts kubectl -n flux-system create secret generic flux-system \ @@ -302,16 +348,7 @@ jobs: # Wait directly on the ESO Helm objects; Kustomization readiness hides useful failure details. wait_for_resource flux-system kustomization.kustomize.toolkit.fluxcd.io/addon-external-secrets 600 kubectl -n flux-system annotate kustomization/addon-external-secrets reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite - wait_for_resource flux-system helmrepository.source.toolkit.fluxcd.io/external-secrets 600 - if ! kubectl -n flux-system wait --for=condition=Ready helmrepository/external-secrets --timeout=900s; then - eso_diagnostics - exit 1 - fi - wait_for_resource flux-system helmrelease.helm.toolkit.fluxcd.io/external-secrets 600 - if ! kubectl -n flux-system wait --for=condition=Ready helmrelease/external-secrets --timeout=1800s; then - eso_diagnostics - exit 1 - fi + wait_for_flux_helm_release external-secrets flux-system-external-secrets external-secrets external-secrets 900s 1800s 1800s wait_for_resource "" crd/clustersecretstores.external-secrets.io 900 wait_for_resource "" crd/externalsecrets.external-secrets.io 900 kubectl wait --for=condition=established --timeout=600s crd/clustersecretstores.external-secrets.io @@ -334,7 +371,9 @@ jobs: namespace: external-secrets EOF # Wait for the storage layer and private access components - kubectl -n flux-system wait --for=condition=Ready kustomization/addon-tailscale-operator --timeout=300s + wait_for_flux_helm_release tailscale flux-system-tailscale-operator tailscale-operator tailscale-system 1200s 1800s 1800s + kubectl -n tailscale-system rollout status deployment/operator --timeout=600s + wait_for_flux_helm_release nfs-subdir-external-provisioner flux-system-nfs-subdir-external-provisioner nfs-subdir-external-provisioner kube-system 1200s 1800s 1800s kubectl -n kube-system rollout status deployment/kube-system-nfs-subdir-external-provisioner --timeout=600s kubectl get storageclass flash-nfs