HetznerTerra/ansible/roles/observability/tasks/main.yml

---
- name: Check if Helm is installed
  command: helm version --short
  register: helm_check
  changed_when: false
  failed_when: false

- name: Install Helm
  shell: curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
  when: helm_check.rc != 0
  changed_when: true

- name: Ensure observability namespace exists
  command: kubectl create namespace {{ observability_namespace }}
  register: create_observability_ns
  failed_when: create_observability_ns.rc != 0 and "AlreadyExists" not in create_observability_ns.stderr
  changed_when: create_observability_ns.rc == 0

- name: Set Grafana admin password
  set_fact:
    grafana_password_effective: "{{ grafana_admin_password if grafana_admin_password | length > 0 else lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}"

- name: Write kube-prometheus-stack values
  copy:
    dest: /tmp/kube-prometheus-stack-values.yaml
    mode: "0644"
    content: |
      grafana:
        enabled: true
        adminPassword: {{ grafana_password_effective }}
        persistence:
          enabled: true
          storageClassName: {{ grafana_storage_class }}
          size: {{ grafana_storage_size }}
        service:
          type: ClusterIP
      prometheus:
        prometheusSpec:
          retention: 7d
          storageSpec:
            volumeClaimTemplate:
              spec:
                storageClassName: {{ prometheus_storage_class }}
                accessModes: ["ReadWriteOnce"]
                resources:
                  requests:
                    storage: {{ prometheus_storage_size }}
      alertmanager:
        enabled: false
      kubeEtcd:
        enabled: false
      kubeControllerManager:
        enabled: false
      kubeScheduler:
        enabled: false

- name: Add Prometheus Helm repo
  command: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
  register: add_prom_repo
  failed_when: add_prom_repo.rc != 0 and "already exists" not in add_prom_repo.stderr
  changed_when: add_prom_repo.rc == 0

- name: Add Grafana Helm repo
  command: helm repo add grafana https://grafana.github.io/helm-charts
  register: add_grafana_repo
  failed_when: add_grafana_repo.rc != 0 and "already exists" not in add_grafana_repo.stderr
  changed_when: add_grafana_repo.rc == 0

- name: Update Helm repos
  command: helm repo update
  changed_when: false

- name: Install kube-prometheus-stack
  command: >-
    helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack
    --namespace {{ observability_namespace }}
    --version {{ prometheus_chart_version }}
    --values /tmp/kube-prometheus-stack-values.yaml
    --wait
    --timeout 10m
  changed_when: true

- name: Write Loki values
  copy:
    dest: /tmp/loki-values.yaml
    mode: "0644"
    content: |
      loki:
        auth_enabled: false
        commonConfig:
          replication_factor: 1
        storage:
          type: filesystem
      singleBinary:
        replicas: 1
        persistence:
          enabled: true
          storageClass: {{ loki_storage_class }}
          size: {{ loki_storage_size }}
      test:
        enabled: false
      monitoring:
        selfMonitoring:
          enabled: false
        lokiCanary:
          enabled: false

- name: Install Loki
  command: >-
    helm upgrade --install loki grafana/loki
    --namespace {{ observability_namespace }}
    --version {{ loki_chart_version }}
    --values /tmp/loki-values.yaml
    --wait
    --timeout 10m
  changed_when: true

- name: Write Promtail values
  copy:
    dest: /tmp/promtail-values.yaml
    mode: "0644"
    content: |
      config:
        clients:
          - url: http://loki-gateway.{{ observability_namespace }}.svc.cluster.local/loki/api/v1/push

- name: Install Promtail
  command: >-
    helm upgrade --install promtail grafana/promtail
    --namespace {{ observability_namespace }}
    --version {{ promtail_chart_version }}
    --values /tmp/promtail-values.yaml
    --wait
    --timeout 10m
  changed_when: true

- name: Create Grafana Loki datasource
  command: kubectl apply -f -
  args:
    stdin: |
      apiVersion: v1
      kind: ConfigMap
      metadata:
        name: grafana-datasource-loki
        namespace: {{ observability_namespace }}
        labels:
          grafana_datasource: "1"
      data:
        loki-datasource.yaml: |
          apiVersion: 1
          datasources:
            - name: Loki
              type: loki
              access: proxy
              url: http://loki-gateway.{{ observability_namespace }}.svc.cluster.local
              isDefault: false
  changed_when: true

- name: Show observability access details
  debug:
    msg: |
      Observability stack deployed.
      Namespace: {{ observability_namespace }}
      Grafana (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-grafana 3000:80
      Prometheus (tailnet): kubectl -n {{ observability_namespace }} port-forward svc/kube-prometheus-stack-prometheus 9090:9090
      Grafana admin password: {{ grafana_password_effective }}