fix: re-enable Loki with v6.10.0 and minimal working config

ansible/roles/observability/defaults/main.yml

@@ -2,7 +2,7 @@
 observability_namespace: "observability"
 
 prometheus_chart_version: "68.4.4"
-loki_chart_version: "6.24.0"
+loki_chart_version: "6.10.0"
 promtail_chart_version: "6.16.6"
 
 grafana_admin_password: ""
@@ -15,4 +15,4 @@ prometheus_storage_class: "local-path"
 grafana_storage_class: "local-path"
 loki_storage_class: "local-path"
 
-loki_enabled: false
+loki_enabled: true

ansible/roles/observability/tasks/main.yml

@@ -52,7 +52,26 @@
     --timeout 10m
   changed_when: true
 
-- name: Remove legacy Loki resources (if present)
+- name: Write Loki values
+  template:
+    src: loki-values.yaml.j2
+    dest: /tmp/loki-values.yaml
+    mode: "0644"
+  when: loki_enabled
+
+- name: Validate Loki chart produces resources
+  command: >-
+    helm template loki grafana/loki
+    --namespace {{ observability_namespace }}
+    --version {{ loki_chart_version }}
+    --values /tmp/loki-values.yaml
+  register: loki_template
+  changed_when: false
+  failed_when: 
+    - loki_template.rc != 0 or 'kind: StatefulSet' not in loki_template.stdout
+  when: loki_enabled
+
+- name: Remove legacy Loki resources
   command: >-
     kubectl -n {{ observability_namespace }} delete
     deployment/loki-gateway
@@ -67,7 +86,48 @@
     --ignore-not-found=true
   changed_when: false
   failed_when: false
-  when: not loki_enabled
+  when: loki_enabled
+
+- name: Install Loki
+  command: >-
+    helm upgrade --install loki grafana/loki
+    --namespace {{ observability_namespace }}
+    --version {{ loki_chart_version }}
+    --values /tmp/loki-values.yaml
+    --wait
+    --timeout 10m
+  changed_when: true
+  when: loki_enabled
+
+- name: Write Promtail values
+  template:
+    src: promtail-values.yaml.j2
+    dest: /tmp/promtail-values.yaml
+    mode: "0644"
+  when: loki_enabled
+
+- name: Install Promtail
+  command: >-
+    helm upgrade --install promtail grafana/promtail
+    --namespace {{ observability_namespace }}
+    --version {{ promtail_chart_version }}
+    --values /tmp/promtail-values.yaml
+    --wait
+    --timeout 10m
+  changed_when: true
+  when: loki_enabled
+
+- name: Write Grafana Loki datasource manifest
+  template:
+    src: grafana-datasource-loki.yaml.j2
+    dest: /tmp/grafana-datasource-loki.yaml
+    mode: "0644"
+  when: loki_enabled
+
+- name: Create Grafana Loki datasource
+  command: kubectl apply -f /tmp/grafana-datasource-loki.yaml
+  changed_when: true
+  when: loki_enabled
 
 - name: Show observability access details
   debug:
@@ -77,4 +137,8 @@
       Grafana (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-grafana 3000:80
       Prometheus (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-prometheus 9090:9090
       Grafana admin password: {{ grafana_password_effective }}
-      Note: Loki logging disabled (set loki_enabled=true to enable)
+      {% if loki_enabled %}
+      Loki: Enabled - logs available in Grafana
+      {% else %}
+      Loki: Disabled
+      {% endif %}

ansible/roles/observability/templates/loki-values.yaml.j2

@@ -4,13 +4,9 @@ loki:
   auth_enabled: false
   commonConfig:
     replication_factor: 1
-  limits_config:
-    retention_period: 168h
-    allow_structured_metadata: true
-    volume_enabled: true
   schemaConfig:
     configs:
-      - from: "2024-01-01"
+      - from: "2024-04-01"
         store: tsdb
         object_store: filesystem
         schema: v13
@@ -19,52 +15,20 @@ loki:
           period: 24h
   storage:
     type: filesystem
+  limits_config:
+    allow_structured_metadata: true
+    volume_enabled: true
+    retention_period: 168h
   pattern_ingester:
     enabled: true
   ruler:
     enable_api: true
 
-backend:
-  replicas: 0
-
-read:
-  replicas: 0
-
-write:
-  replicas: 0
-
-ingester:
-  replicas: 0
-
-querier:
-  replicas: 0
-
-queryFrontend:
-  replicas: 0
-
-queryScheduler:
-  replicas: 0
-
-distributor:
-  replicas: 0
-
-compactor:
-  replicas: 0
-
-indexGateway:
-  replicas: 0
-
-bloomCompactor:
-  replicas: 0
-
-bloomGateway:
-  replicas: 0
-
-gateway:
-  enabled: false
-
 singleBinary:
   replicas: 1
+  persistence:
+    size: {{ loki_storage_size }}
+    storageClass: {{ loki_storage_class }}
   resources:
     requests:
       cpu: 100m
@@ -72,10 +36,34 @@ singleBinary:
     limits:
       cpu: 500m
       memory: 1Gi
-  persistence:
-    enabled: true
-    storageClass: {{ loki_storage_class }}
-    size: {{ loki_storage_size }}
+
+backend:
+  replicas: 0
+read:
+  replicas: 0
+write:
+  replicas: 0
+ingester:
+  replicas: 0
+querier:
+  replicas: 0
+queryFrontend:
+  replicas: 0
+queryScheduler:
+  replicas: 0
+distributor:
+  replicas: 0
+compactor:
+  replicas: 0
+indexGateway:
+  replicas: 0
+bloomCompactor:
+  replicas: 0
+bloomGateway:
+  replicas: 0
+
+gateway:
+  enabled: false
 
 test:
   enabled: false