From 480a079dc8d5863050a30cdde4c8dd33278b9488 Mon Sep 17 00:00:00 2001
From: MichaelFisher1997 <contact@michaelfisher.tech>
Date: Wed, 4 Mar 2026 21:00:01 +0000
Subject: [PATCH] fix: fail fast when loki datasource has no labels

---
 .../observability-content/tasks/main.yml      | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ansible/roles/observability-content/tasks/main.yml b/ansible/roles/observability-content/tasks/main.yml
index 17e329e..6b5379e 100644
--- a/ansible/roles/observability-content/tasks/main.yml
+++ b/ansible/roles/observability-content/tasks/main.yml
@@ -98,6 +98,26 @@
     - loki_host_ip.stdout | length > 0
     - loki_nodeport.stdout | length > 0
 
+- name: Query Loki labels endpoint from Grafana pod
+  shell: >-
+    kubectl -n {{ observability_namespace }} exec {{ grafana_pod_name.stdout }} -c grafana --
+    sh -c 'wget -qO- --timeout=10 {{ grafana_loki_effective_url }}/loki/api/v1/labels'
+  register: grafana_loki_labels
+  changed_when: false
+  failed_when: false
+  when: loki_enabled
+
+- name: Fail when Loki is reachable but has zero indexed labels
+  fail:
+    msg: >-
+      Loki is reachable from Grafana at {{ grafana_loki_effective_url }} but /loki/api/v1/labels returned no labels.
+      This usually means no logs are ingested yet. Check Promtail and tenant configuration.
+  when:
+    - loki_enabled
+    - grafana_loki_labels.rc == 0
+    - "'\"status\":\"success\"' in (grafana_loki_labels.stdout | replace(' ', ''))"
+    - "'\"data\":[]' in (grafana_loki_labels.stdout | replace(' ', ''))"
+
 - name: Write default Prometheus datasource ConfigMap patch
   template:
     src: grafana-default-prometheus-datasource.yaml.j2