fix: harden terraform destroy workflow

.gitea/workflows/destroy.yml

@@ -51,11 +51,57 @@ jobs:
           echo "${{ secrets.SSH_PUBLIC_KEY }}" > ~/.ssh/id_ed25519.pub
           chmod 644 ~/.ssh/id_ed25519.pub
 
+      - name: Install jq
+        run: |
+          apt-get update
+          apt-get install -y jq
+
       - name: Terraform Destroy
+        id: destroy
         working-directory: terraform
         run: |
-          terraform destroy \
+          set +e
-            -var="hcloud_token=${{ secrets.HCLOUD_TOKEN }}" \
+          for attempt in 1 2 3; do
-            -var="ssh_public_key=$HOME/.ssh/id_ed25519.pub" \
+            echo "Terraform destroy attempt ${attempt}/3"
-            -var="ssh_private_key=$HOME/.ssh/id_ed25519" \
+            terraform destroy \
-            -auto-approve
+              -var="hcloud_token=${{ secrets.HCLOUD_TOKEN }}" \
+              -var="ssh_public_key=$HOME/.ssh/id_ed25519.pub" \
+              -var="ssh_private_key=$HOME/.ssh/id_ed25519" \
+              -auto-approve
+            rc=$?
+            if [ "$rc" -eq 0 ]; then
+              exit 0
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              echo "Terraform destroy failed with exit code ${rc}; retrying in 30s"
+              sleep 30
+              terraform refresh \
+                -var="hcloud_token=${{ secrets.HCLOUD_TOKEN }}" \
+                -var="ssh_public_key=$HOME/.ssh/id_ed25519.pub" \
+                -var="ssh_private_key=$HOME/.ssh/id_ed25519" || true
+            fi
+          done
+          exit "$rc"
+
+      - name: Hetzner destroy diagnostics
+        if: failure() && steps.destroy.outcome == 'failure'
+        env:
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+        run: |
+          set +e
+          echo "== Terraform state list =="
+          terraform -chdir=terraform state list || true
+
+          network_id=$(terraform -chdir=terraform state show hcloud_network.cluster 2>/dev/null | awk '/^id *=/ {gsub(/"/, "", $3); print $3; exit}')
+          if [ -z "$network_id" ]; then
+            network_id="11988935"
+          fi
+
+          echo "== Hetzner network =="
+          curl -fsSL -H "Authorization: Bearer ${HCLOUD_TOKEN}" "https://api.hetzner.cloud/v1/networks/${network_id}" | jq . || true
+
+          echo "== Hetzner servers attached to network =="
+          curl -fsSL -H "Authorization: Bearer ${HCLOUD_TOKEN}" "https://api.hetzner.cloud/v1/servers" | jq --argjson id "$network_id" '.servers[] | select(any(.private_net[]?; .network == $id)) | {id, name, private_net}' || true
+
+          echo "== Hetzner load balancers attached to network =="
+          curl -fsSL -H "Authorization: Bearer ${HCLOUD_TOKEN}" "https://api.hetzner.cloud/v1/load_balancers" | jq --argjson id "$network_id" '.load_balancers[] | select(any(.private_net[]?; .network == $id)) | {id, name, private_net}' || true