fix: add loki nodeport fallback for grafana datasource reachability

ansible/roles/observability-content/defaults/main.yml

@@ -6,3 +6,4 @@ loki_enabled: true
 grafana_prometheus_url: "http://kube-prometheus-stack-prometheus.{{ observability_namespace }}.svc.cluster.local:9090"
 grafana_loki_url: "http://loki.{{ observability_namespace }}.svc.cluster.local:3100"
 grafana_use_prometheus_nodeport_fallback: true
+grafana_use_loki_nodeport_fallback: true

ansible/roles/observability-content/tasks/main.yml

@@ -12,6 +12,7 @@
 - name: Set default Prometheus datasource URL
   set_fact:
     grafana_prometheus_effective_url: "{{ grafana_prometheus_url }}"
+    grafana_loki_effective_url: "{{ grafana_loki_url }}"
 
 - name: Get Grafana pod name
   command: kubectl -n {{ observability_namespace }} get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}'
@@ -26,6 +27,15 @@
   changed_when: false
   failed_when: false
 
+- name: Probe Loki from Grafana pod via default datasource URL
+  shell: >-
+    kubectl -n {{ observability_namespace }} exec {{ grafana_pod_name.stdout }} -c grafana --
+    sh -c 'wget -qO- --timeout=5 {{ grafana_loki_url }}/ready >/dev/null'
+  register: grafana_loki_probe
+  changed_when: false
+  failed_when: false
+  when: loki_enabled
+
 - name: Get Prometheus pod host IP for fallback
   command: kubectl -n {{ observability_namespace }} get pod -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].status.hostIP}'
   register: prometheus_host_ip
@@ -51,6 +61,43 @@
     - prometheus_host_ip.stdout | length > 0
     - prometheus_nodeport.stdout | length > 0
 
+- name: Ensure Loki service uses NodePort for fallback
+  command: kubectl -n {{ observability_namespace }} patch svc loki -p '{"spec":{"type":"NodePort"}}'
+  changed_when: false
+  failed_when: false
+  when:
+    - loki_enabled
+    - grafana_use_loki_nodeport_fallback | bool
+    - grafana_loki_probe.rc != 0
+
+- name: Get Loki pod host IP for fallback
+  command: kubectl -n {{ observability_namespace }} get pod loki-0 -o jsonpath='{.status.hostIP}'
+  register: loki_host_ip
+  changed_when: false
+  when:
+    - loki_enabled
+    - grafana_use_loki_nodeport_fallback | bool
+    - grafana_loki_probe.rc != 0
+
+- name: Get Loki service NodePort for fallback
+  command: kubectl -n {{ observability_namespace }} get svc loki -o jsonpath='{.spec.ports[?(@.name=="http-metrics")].nodePort}'
+  register: loki_nodeport
+  changed_when: false
+  when:
+    - loki_enabled
+    - grafana_use_loki_nodeport_fallback | bool
+    - grafana_loki_probe.rc != 0
+
+- name: Enable Loki NodePort fallback datasource URL
+  set_fact:
+    grafana_loki_effective_url: "http://{{ loki_host_ip.stdout }}:{{ loki_nodeport.stdout }}"
+  when:
+    - loki_enabled
+    - grafana_use_loki_nodeport_fallback | bool
+    - grafana_loki_probe.rc != 0
+    - loki_host_ip.stdout | length > 0
+    - loki_nodeport.stdout | length > 0
+
 - name: Write default Prometheus datasource ConfigMap patch
   template:
     src: grafana-default-prometheus-datasource.yaml.j2
@@ -102,4 +149,5 @@
       Grafana content applied.
       Datasources ConfigMap: {{ grafana_datasource_configmap_name }}
       Prometheus datasource URL: {{ grafana_prometheus_effective_url }}
+      Loki datasource URL: {{ grafana_loki_effective_url }}
       Dashboard ConfigMap: {{ grafana_dashboard_configmap_name }}

ansible/roles/observability-content/templates/grafana-datasources.yaml.j2

@@ -13,6 +13,6 @@ data:
       - name: Loki
         type: loki
         access: proxy
-        url: "{{ grafana_loki_url }}"
+        url: "{{ grafana_loki_effective_url }}"
         isDefault: false
 {% endif %}

ansible/roles/observability-content/templates/grafana-default-prometheus-datasource.yaml.j2

@@ -10,7 +10,7 @@ data:
     - name: "Prometheus"
       type: prometheus
       uid: prometheus
-      url: {{ grafana_prometheus_effective_url }}/
+      url: "{{ grafana_prometheus_effective_url }}/"
       access: proxy
       isDefault: true
       jsonData: