From 124fe94d0ea9c56371395e68438c8633c627bb74 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Mon, 2 Mar 2026 09:12:47 +0000 Subject: [PATCH] fix: tolerate Loki rollout false negatives when core pod is ready --- ansible/roles/observability/tasks/main.yml | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml index 36f0f7b..638764a 100644 --- a/ansible/roles/observability/tasks/main.yml +++ b/ansible/roles/observability/tasks/main.yml @@ -66,6 +66,19 @@ --values /tmp/loki-values.yaml changed_when: false +- name: Remove legacy Loki scalable workloads (if present) + command: >- + kubectl -n {{ observability_namespace }} delete + deployment/loki-gateway + statefulset/loki-chunks-cache + statefulset/loki-results-cache + statefulset/loki-backend + statefulset/loki-read + statefulset/loki-write + --ignore-not-found=true + changed_when: false + failed_when: false + - name: Install Loki command: >- helm upgrade --install loki grafana/loki @@ -84,6 +97,13 @@ changed_when: false when: loki_install.rc == 0 +- name: Check Loki core container readiness + command: kubectl -n {{ observability_namespace }} get pod loki-0 -o jsonpath={.status.containerStatuses[?(@.name=="loki")].ready} + register: loki_core_ready + failed_when: false + changed_when: false + when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 + - name: Show Loki pods on install failure command: kubectl -n {{ observability_namespace }} get pods -l app.kubernetes.io/name=loki -o wide register: loki_pods_status @@ -154,7 +174,9 @@ Events: {{ loki_events.stdout | default('n/a') }} - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 + when: + - loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 + - loki_core_ready.stdout | default('false') != 'true' - name: Write Promtail values template: