feat: manage grafana content as code with fast dashboard workflow

ansible/roles/observability/tasks/main.yml

@@ -52,6 +52,17 @@
     --timeout 10m
   changed_when: true
 
+- name: Wait for Grafana deployment rollout
+  command: kubectl -n {{ observability_namespace }} rollout status deployment/kube-prometheus-stack-grafana --timeout=5m
+  changed_when: false
+
+- name: Reset Grafana admin password in Grafana database
+  shell: >-
+    kubectl -n {{ observability_namespace }} exec
+    "$(kubectl -n {{ observability_namespace }} get pod -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}')"
+    -c grafana -- grafana cli admin reset-admin-password '{{ grafana_password_effective }}'
+  changed_when: true
+
 - name: Write Loki values
   template:
     src: loki-values.yaml.j2
@@ -144,18 +155,6 @@
   changed_when: true
   when: loki_enabled
 
-- name: Write Grafana Loki datasource manifest
-  template:
-    src: grafana-datasource-loki.yaml.j2
-    dest: /tmp/grafana-datasource-loki.yaml
-    mode: "0644"
-  when: loki_enabled
-
-- name: Create Grafana Loki datasource
-  command: kubectl apply -f /tmp/grafana-datasource-loki.yaml
-  changed_when: true
-  when: loki_enabled
-
 - name: Check Tailscale service readiness for Grafana
   command: kubectl -n {{ observability_namespace }} get svc kube-prometheus-stack-grafana -o jsonpath='{.status.conditions[?(@.type=="TailscaleProxyReady")].status}'
   register: grafana_tailscale_ready