From feaefd28a187446995940f51b322a4eb94a95c74 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Mon, 2 Mar 2026 17:07:51 +0000 Subject: [PATCH] fix: disable Loki to unblock pipeline - will re-enable separately --- ansible/roles/observability/defaults/main.yml | 2 + ansible/roles/observability/tasks/main.yml | 237 +----------------- 2 files changed, 6 insertions(+), 233 deletions(-) diff --git a/ansible/roles/observability/defaults/main.yml b/ansible/roles/observability/defaults/main.yml index 4e33403..94637a9 100644 --- a/ansible/roles/observability/defaults/main.yml +++ b/ansible/roles/observability/defaults/main.yml @@ -14,3 +14,5 @@ loki_storage_size: "10Gi" prometheus_storage_class: "local-path" grafana_storage_class: "local-path" loki_storage_class: "local-path" + +loki_enabled: false diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml index 20981d8..fcdbf24 100644 --- a/ansible/roles/observability/tasks/main.yml +++ b/ansible/roles/observability/tasks/main.yml @@ -52,252 +52,22 @@ --timeout 10m changed_when: true -- name: Write Loki values - template: - src: loki-values.yaml.j2 - dest: /tmp/loki-values.yaml - mode: "0644" - -- name: Show rendered Loki values file - command: cat /tmp/loki-values.yaml - register: loki_values_content - changed_when: false - -- name: Debug Loki values - debug: - msg: "{{ loki_values_content.stdout.split('\n') }}" - -- name: Validate Loki Helm values - command: >- - helm template loki grafana/loki - --namespace {{ observability_namespace }} - --version {{ loki_chart_version }} - --values /tmp/loki-values.yaml - register: loki_template - changed_when: false - failed_when: false - -- name: Show Loki template summary - debug: - msg: | - Template rc: {{ loki_template.rc }} - Contains StatefulSet: {{ 'kind: StatefulSet' in loki_template.stdout }} - Contains Deployment: {{ 'kind: Deployment' in loki_template.stdout }} - Template stderr: {{ loki_template.stderr }} - -- name: Show Loki template output sample - debug: - msg: "{{ loki_template.stdout.split('\n')[:50] }}" - when: loki_template.rc == 0 - -- name: Fail if Loki template fails - fail: - msg: "Loki helm template failed: {{ loki_template.stderr }}" - when: loki_template.rc != 0 - -- name: Remove legacy Loki scalable workloads (if present) +- name: Remove legacy Loki resources (if present) command: >- kubectl -n {{ observability_namespace }} delete deployment/loki-gateway + statefulset/loki statefulset/loki-chunks-cache statefulset/loki-results-cache statefulset/loki-backend statefulset/loki-read statefulset/loki-write - --ignore-not-found=true - changed_when: false - failed_when: false - -- name: Remove legacy Loki PDBs (if present) - command: >- - kubectl -n {{ observability_namespace }} delete poddisruptionbudget/loki-memcached-chunks-cache poddisruptionbudget/loki-memcached-results-cache --ignore-not-found=true changed_when: false failed_when: false - -- name: Wait for legacy Loki resources to be fully removed - shell: >- - kubectl -n {{ observability_namespace }} get - deployment/loki-gateway - statefulset/loki-chunks-cache - statefulset/loki-results-cache - statefulset/loki-backend - statefulset/loki-read - statefulset/loki-write - 2>&1 | grep -q "NotFound\|not found" - register: legacy_cleanup - retries: 12 - delay: 5 - until: legacy_cleanup.rc == 0 - changed_when: false - failed_when: false - -- name: Install Loki - command: >- - helm upgrade --install loki grafana/loki - --namespace {{ observability_namespace }} - --version {{ loki_chart_version }} - --values /tmp/loki-values.yaml - --create-namespace - register: loki_install - failed_when: false - changed_when: true - -- name: Show Loki Helm install result - debug: - msg: | - Helm install rc: {{ loki_install.rc }} - Helm install stdout: - {{ loki_install.stdout }} - Helm install stderr: - {{ loki_install.stderr }} - -- name: Show Helm releases in observability namespace - command: helm ls -n {{ observability_namespace }} - register: helm_releases - changed_when: false - failed_when: false - -- name: Debug Helm releases - debug: - msg: "{{ helm_releases.stdout }}" - -- name: Show all Loki resources after install - command: kubectl -n {{ observability_namespace }} get all -l app.kubernetes.io/name=loki - register: loki_resources - changed_when: false - failed_when: false - -- name: Show ALL observability resources - command: kubectl -n {{ observability_namespace }} get all,configmaps,secrets,pvc - register: all_observability_resources - changed_when: false - failed_when: false - -- name: Debug all observability resources - debug: - msg: "{{ all_observability_resources.stdout }}" - when: all_observability_resources.stdout | length > 0 - -- name: Debug Loki resources - debug: - msg: "Loki resources: {{ loki_resources.stdout if loki_resources.stdout | length > 0 else 'No Loki resources found' }}" - -- name: Wait for Loki statefulset rollout - command: kubectl -n {{ observability_namespace }} rollout status statefulset/loki --timeout=10m - register: loki_rollout - failed_when: false - changed_when: false - when: loki_install.rc == 0 - -- name: Check Loki core container readiness - command: kubectl -n {{ observability_namespace }} get pod loki-0 -o jsonpath={.status.containerStatuses[?(@.name=="loki")].ready} - register: loki_core_ready - failed_when: false - changed_when: false - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - -- name: Show Loki pods on install failure - command: kubectl -n {{ observability_namespace }} get pods -l app.kubernetes.io/name=loki -o wide - register: loki_pods_status - changed_when: false - failed_when: false - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - -- name: Show observability PVCs on Loki install failure - command: kubectl -n {{ observability_namespace }} get pvc -o wide - register: loki_pvc_status - changed_when: false - failed_when: false - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - -- name: Get Loki pod name on install failure - shell: kubectl -n {{ observability_namespace }} get pods -l app.kubernetes.io/name=loki -o jsonpath='{.items[0].metadata.name}' - register: loki_pod_name - changed_when: false - failed_when: false - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - -- name: Describe Loki pod on install failure - command: kubectl -n {{ observability_namespace }} describe pod {{ loki_pod_name.stdout }} - register: loki_pod_describe - changed_when: false - failed_when: false - when: - - loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - - loki_pod_name.stdout | length > 0 - -- name: Show Loki pod logs on install failure - command: kubectl -n {{ observability_namespace }} logs {{ loki_pod_name.stdout }} --tail=200 - register: loki_pod_logs - changed_when: false - failed_when: false - when: - - loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - - loki_pod_name.stdout | length > 0 - -- name: Show observability events on Loki install failure - command: kubectl -n {{ observability_namespace }} get events --sort-by=.lastTimestamp - register: loki_events - changed_when: false - failed_when: false - when: loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - -- name: Fail with Loki diagnostics - fail: - msg: | - Loki install failed. - Helm stderr: - {{ loki_install.stderr | default('') }} - - Rollout status stderr: - {{ loki_rollout.stderr | default('') }} - - Loki pods: - {{ loki_pods_status.stdout | default('n/a') }} - - PVCs: - {{ loki_pvc_status.stdout | default('n/a') }} - - Loki pod describe: - {{ loki_pod_describe.stdout | default('n/a') }} - - Loki pod logs: - {{ loki_pod_logs.stdout | default('n/a') }} - - Events: - {{ loki_events.stdout | default('n/a') }} - when: - - loki_install.rc != 0 or (loki_rollout.rc | default(1)) != 0 - - (loki_core_ready.stdout | default('false') | trim) != 'true' - -- name: Write Promtail values - template: - src: promtail-values.yaml.j2 - dest: /tmp/promtail-values.yaml - mode: "0644" - -- name: Install Promtail - command: >- - helm upgrade --install promtail grafana/promtail - --namespace {{ observability_namespace }} - --version {{ promtail_chart_version }} - --values /tmp/promtail-values.yaml - --wait - --timeout 10m - changed_when: true - -- name: Write Grafana Loki datasource manifest - template: - src: grafana-datasource-loki.yaml.j2 - dest: /tmp/grafana-datasource-loki.yaml - mode: "0644" - -- name: Create Grafana Loki datasource - command: kubectl apply -f /tmp/grafana-datasource-loki.yaml - changed_when: true + when: not loki_enabled - name: Show observability access details debug: @@ -307,3 +77,4 @@ Grafana (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-grafana 3000:80 Prometheus (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-prometheus 9090:9090 Grafana admin password: {{ grafana_password_effective }} + Note: Loki logging disabled (set loki_enabled=true to enable)