fix: clear stale helm pending revisions before kube-prometheus upgrade

ansible/roles/observability/tasks/main.yml

@@ -42,6 +42,20 @@
   command: helm repo update
   changed_when: false
 
+- name: Clear stale pending Helm revision secrets for kube-prometheus-stack
+  shell: >-
+    kubectl -n {{ observability_namespace }} delete
+    $(kubectl -n {{ observability_namespace }} get secret -l owner=helm,name=kube-prometheus-stack,status=pending-upgrade -o name)
+    --ignore-not-found=true;
+    kubectl -n {{ observability_namespace }} delete
+    $(kubectl -n {{ observability_namespace }} get secret -l owner=helm,name=kube-prometheus-stack,status=pending-install -o name)
+    --ignore-not-found=true;
+    kubectl -n {{ observability_namespace }} delete
+    $(kubectl -n {{ observability_namespace }} get secret -l owner=helm,name=kube-prometheus-stack,status=pending-rollback -o name)
+    --ignore-not-found=true
+  changed_when: false
+  failed_when: false
+
 - name: Install kube-prometheus-stack
   command: >-
     helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack
@@ -51,8 +65,8 @@
     --wait
     --timeout 10m
   register: kube_prom_install
-  retries: 6
+  retries: 12
-  delay: 20
+  delay: 15
   until: kube_prom_install.rc == 0
   changed_when: true