diff --git a/ansible/roles/observability-content/defaults/main.yml b/ansible/roles/observability-content/defaults/main.yml index c970e72..2164bb0 100644 --- a/ansible/roles/observability-content/defaults/main.yml +++ b/ansible/roles/observability-content/defaults/main.yml @@ -6,3 +6,4 @@ loki_enabled: true grafana_prometheus_url: "http://kube-prometheus-stack-prometheus.{{ observability_namespace }}.svc.cluster.local:9090" grafana_loki_url: "http://loki.{{ observability_namespace }}.svc.cluster.local:3100" grafana_use_prometheus_nodeport_fallback: true +grafana_use_loki_nodeport_fallback: true diff --git a/ansible/roles/observability-content/tasks/main.yml b/ansible/roles/observability-content/tasks/main.yml index 1e2fbfa..17e329e 100644 --- a/ansible/roles/observability-content/tasks/main.yml +++ b/ansible/roles/observability-content/tasks/main.yml @@ -12,6 +12,7 @@ - name: Set default Prometheus datasource URL set_fact: grafana_prometheus_effective_url: "{{ grafana_prometheus_url }}" + grafana_loki_effective_url: "{{ grafana_loki_url }}" - name: Get Grafana pod name command: kubectl -n {{ observability_namespace }} get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}' @@ -26,6 +27,15 @@ changed_when: false failed_when: false +- name: Probe Loki from Grafana pod via default datasource URL + shell: >- + kubectl -n {{ observability_namespace }} exec {{ grafana_pod_name.stdout }} -c grafana -- + sh -c 'wget -qO- --timeout=5 {{ grafana_loki_url }}/ready >/dev/null' + register: grafana_loki_probe + changed_when: false + failed_when: false + when: loki_enabled + - name: Get Prometheus pod host IP for fallback command: kubectl -n {{ observability_namespace }} get pod -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].status.hostIP}' register: prometheus_host_ip @@ -51,6 +61,43 @@ - prometheus_host_ip.stdout | length > 0 - prometheus_nodeport.stdout | length > 0 +- name: Ensure Loki service uses NodePort for fallback + command: kubectl -n {{ observability_namespace }} patch svc loki -p '{"spec":{"type":"NodePort"}}' + changed_when: false + failed_when: false + when: + - loki_enabled + - grafana_use_loki_nodeport_fallback | bool + - grafana_loki_probe.rc != 0 + +- name: Get Loki pod host IP for fallback + command: kubectl -n {{ observability_namespace }} get pod loki-0 -o jsonpath='{.status.hostIP}' + register: loki_host_ip + changed_when: false + when: + - loki_enabled + - grafana_use_loki_nodeport_fallback | bool + - grafana_loki_probe.rc != 0 + +- name: Get Loki service NodePort for fallback + command: kubectl -n {{ observability_namespace }} get svc loki -o jsonpath='{.spec.ports[?(@.name=="http-metrics")].nodePort}' + register: loki_nodeport + changed_when: false + when: + - loki_enabled + - grafana_use_loki_nodeport_fallback | bool + - grafana_loki_probe.rc != 0 + +- name: Enable Loki NodePort fallback datasource URL + set_fact: + grafana_loki_effective_url: "http://{{ loki_host_ip.stdout }}:{{ loki_nodeport.stdout }}" + when: + - loki_enabled + - grafana_use_loki_nodeport_fallback | bool + - grafana_loki_probe.rc != 0 + - loki_host_ip.stdout | length > 0 + - loki_nodeport.stdout | length > 0 + - name: Write default Prometheus datasource ConfigMap patch template: src: grafana-default-prometheus-datasource.yaml.j2 @@ -102,4 +149,5 @@ Grafana content applied. Datasources ConfigMap: {{ grafana_datasource_configmap_name }} Prometheus datasource URL: {{ grafana_prometheus_effective_url }} + Loki datasource URL: {{ grafana_loki_effective_url }} Dashboard ConfigMap: {{ grafana_dashboard_configmap_name }} diff --git a/ansible/roles/observability-content/templates/grafana-datasources.yaml.j2 b/ansible/roles/observability-content/templates/grafana-datasources.yaml.j2 index bdf09d3..ffd0b6b 100644 --- a/ansible/roles/observability-content/templates/grafana-datasources.yaml.j2 +++ b/ansible/roles/observability-content/templates/grafana-datasources.yaml.j2 @@ -13,6 +13,6 @@ data: - name: Loki type: loki access: proxy - url: "{{ grafana_loki_url }}" + url: "{{ grafana_loki_effective_url }}" isDefault: false {% endif %} diff --git a/ansible/roles/observability-content/templates/grafana-default-prometheus-datasource.yaml.j2 b/ansible/roles/observability-content/templates/grafana-default-prometheus-datasource.yaml.j2 index 956ecc7..dfbf3f6 100644 --- a/ansible/roles/observability-content/templates/grafana-default-prometheus-datasource.yaml.j2 +++ b/ansible/roles/observability-content/templates/grafana-default-prometheus-datasource.yaml.j2 @@ -10,7 +10,7 @@ data: - name: "Prometheus" type: prometheus uid: prometheus - url: {{ grafana_prometheus_effective_url }}/ + url: "{{ grafana_prometheus_effective_url }}/" access: proxy isDefault: true jsonData: