feat: Add HA Kubernetes cluster with Terraform + Ansible

- 3x CX23 control plane nodes (HA)
- 4x CX33 worker nodes
- k3s with embedded etcd
- Hetzner CCM for load balancers
- Gitea CI/CD workflows
- Backblaze B2 for Terraform state

ansible/ansible.cfg

@@ -0,0 +1,14 @@
+[defaults]
+inventory = inventory.ini
+host_key_checking = False
+private_key_file = {{ private_key_file }}
+retry_files_enabled = False
+roles_path = roles
+stdout_callback = yaml
+interpreter_python = auto_silent
+
+[privilege_escalation]
+become = True
+become_method = sudo
+become_user = root
+become_ask_pass = False

ansible/generate_inventory.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+
+def get_terraform_outputs():
+    result = subprocess.run(
+        ["terraform", "output", "-json"],
+        cwd="../terraform",
+        capture_output=True,
+        text=True,
+    )
+
+    if result.returncode != 0:
+        print(f"Error running terraform output: {result.stderr}")
+        sys.exit(1)
+
+    return json.loads(result.stdout)
+
+
+def main():
+    outputs = get_terraform_outputs()
+
+    data = {
+        "control_plane_ips": outputs["control_plane_ips"]["value"],
+        "worker_ips": outputs["worker_ips"]["value"],
+        "private_key_file": outputs["ssh_private_key_path"]["value"],
+    }
+
+    env = Environment(loader=FileSystemLoader("."))
+    template = env.get_template("inventory.tmpl")
+    inventory = template.render(**data)
+
+    Path("inventory.ini").write_text(inventory)
+    print("Generated inventory.ini")
+
+
+if __name__ == "__main__":
+    main()

ansible/inventory.tmpl

@@ -0,0 +1,18 @@
+[control_plane]
+{% for ip in control_plane_ips %}
+{{ ip }}
+{% endfor %}
+
+[workers]
+{% for ip in worker_ips %}
+{{ ip }}
+{% endfor %}
+
+[cluster:children]
+control_plane
+workers
+
+[cluster:vars]
+ansible_user=root
+ansible_python_interpreter=/usr/bin/python3
+k3s_version=latest

ansible/requirements.yml

@@ -0,0 +1,5 @@
+collections:
+  - name: kubernetes.core
+    version: ">=2.4.0"
+  - name: community.general
+    version: ">=8.0.0"

ansible/roles/ccm/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+hcloud_token: ""
+cluster_name: "k8s-cluster"

ansible/roles/ccm/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: Check if Hetzner CCM is already deployed
+  command: kubectl get namespace hetzner-cloud-system
+  register: ccm_namespace
+  failed_when: false
+  changed_when: false
+
+- name: Create Hetzner CCM namespace
+  command: kubectl create namespace hetzner-cloud-system
+  when: ccm_namespace.rc != 0
+  changed_when: true
+
+- name: Create Hetzner cloud secret
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: hcloud
+        namespace: hetzner-cloud-system
+      stringData:
+        token: "{{ hcloud_token }}"
+        network: "{{ cluster_name }}-network"
+  no_log: true
+  when: hcloud_token is defined
+
+- name: Deploy Hetzner CCM
+  kubernetes.core.k8s:
+    state: present
+    src: "{{ item }}"
+  loop:
+    - https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/main/deploy/ccm-networks.yaml
+  when: ccm_namespace.rc != 0
+
+- name: Wait for CCM pods to be ready
+  command: kubectl rollout status deployment/hcloud-cloud-controller-manager -n hetzner-cloud-system
+  changed_when: false
+  retries: 30
+  delay: 10

ansible/roles/common/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+common_upgrade_packages: false

ansible/roles/common/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Update apt cache
+  apt:
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Upgrade packages
+  apt:
+    upgrade: dist
+  when: common_upgrade_packages | default(false)
+
+- name: Install required packages
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - curl
+      - gnupg
+      - lsb-release
+      - software-properties-common
+      - jq
+      - htop
+      - vim
+    state: present
+
+- name: Disable swap
+  command: swapoff -a
+  changed_when: true
+
+- name: Remove swap from fstab
+  mount:
+    name: swap
+    fstype: swap
+    state: absent
+
+- name: Load br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+
+- name: Persist br_netfilter module
+  copy:
+    dest: /etc/modules-load.d/k8s.conf
+    content: |
+      br_netfilter
+      overlay
+    mode: "0644"
+
+- name: Configure sysctl for Kubernetes
+  sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: true
+  loop:
+    - { name: net.bridge.bridge-nf-call-iptables, value: 1 }
+    - { name: net.bridge.bridge-nf-call-ip6tables, value: 1 }
+    - { name: net.ipv4.ip_forward, value: 1 }

ansible/roles/k3s-agent/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+k3s_version: latest
+k3s_server_url: ""
+k3s_token: ""

ansible/roles/k3s-agent/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Check if k3s agent is already installed
+  stat:
+    path: /usr/local/bin/k3s-agent
+  register: k3s_agent_binary
+
+- name: Download k3s install script
+  get_url:
+    url: https://get.k3s.io
+    dest: /tmp/install-k3s.sh
+    mode: "0755"
+  when: not k3s_agent_binary.stat.exists
+
+- name: Install k3s agent
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version if k3s_version != 'latest' else '' }}"
+    K3S_URL: "{{ k3s_server_url }}"
+    K3S_TOKEN: "{{ k3s_token }}"
+  command: /tmp/install-k3s.sh agent
+  args:
+    creates: /usr/local/bin/k3s-agent
+  when: not k3s_agent_binary.stat.exists
+
+- name: Wait for k3s agent to be ready
+  command: systemctl is-active k3s-agent
+  register: agent_status
+  until: agent_status.stdout == "active"
+  retries: 30
+  delay: 10
+  changed_when: false

ansible/roles/k3s-server/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+k3s_version: latest
+k3s_token: ""

ansible/roles/k3s-server/tasks/main.yml

@@ -0,0 +1,56 @@
+---
+- name: Check if k3s is already installed
+  stat:
+    path: /usr/local/bin/k3s
+  register: k3s_binary
+
+- name: Download k3s install script
+  get_url:
+    url: https://get.k3s.io
+    dest: /tmp/install-k3s.sh
+    mode: "0755"
+  when: not k3s_binary.stat.exists
+
+- name: Install k3s server (primary)
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version if k3s_version != 'latest' else '' }}"
+    K3S_TOKEN: "{{ k3s_token }}"
+  command: /tmp/install-k3s.sh server --cluster-init
+  args:
+    creates: /usr/local/bin/k3s
+  when: 
+    - not k3s_binary.stat.exists
+    - k3s_primary | default(false)
+
+- name: Install k3s server (secondary)
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version if k3s_version != 'latest' else '' }}"
+    K3S_TOKEN: "{{ k3s_token }}"
+  command: /tmp/install-k3s.sh server --server https://{{ k3s_primary_ip }}:6443
+  args:
+    creates: /usr/local/bin/k3s
+  when: 
+    - not k3s_binary.stat.exists
+    - not (k3s_primary | default(false))
+
+- name: Wait for k3s to be ready
+  command: kubectl get nodes
+  register: k3s_ready
+  until: k3s_ready.rc == 0
+  retries: 30
+  delay: 10
+  changed_when: false
+
+- name: Copy kubeconfig to default location for root
+  file:
+    src: /etc/rancher/k3s/k3s.yaml
+    dest: /root/.kube/config
+    state: link
+    force: true
+
+- name: Ensure .kube directory exists for ansible user
+  file:
+    path: "/home/{{ ansible_user }}/.kube"
+    state: directory
+    mode: "0755"
+  when: ansible_user != 'root'

ansible/site.yml

@@ -0,0 +1,94 @@
+---
+- name: Bootstrap Kubernetes cluster
+  hosts: cluster
+  become: true
+  gather_facts: true
+
+  pre_tasks:
+    - name: Wait for SSH
+      wait_for_connection:
+        delay: 10
+        timeout: 300
+
+  roles:
+    - common
+
+- name: Setup primary control plane
+  hosts: control_plane[0]
+  become: true
+
+  vars:
+    k3s_primary: true
+    k3s_token: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}"
+
+  roles:
+    - k3s-server
+
+- name: Get join info from primary
+  hosts: control_plane[0]
+  become: true
+  tasks:
+    - name: Fetch node token
+      command: cat /var/lib/rancher/k3s/server/node-token
+      register: node_token
+      changed_when: false
+
+    - name: Set join token fact
+      set_fact:
+        k3s_token: "{{ node_token.stdout }}"
+        k3s_primary_ip: "{{ ansible_default_ipv4.address }}"
+
+    - name: Fetch kubeconfig
+      fetch:
+        src: /etc/rancher/k3s/k3s.yaml
+        dest: ../outputs/kubeconfig
+        flat: true
+
+- name: Setup secondary control planes
+  hosts: control_plane[1:]
+  become: true
+
+  vars:
+    k3s_primary: false
+    k3s_token: "{{ hostvars[groups['control_plane'][0]]['k3s_token'] }}"
+    k3s_primary_ip: "{{ hostvars[groups['control_plane'][0]]['ansible_default_ipv4']['address'] }}"
+
+  roles:
+    - k3s-server
+
+- name: Setup workers
+  hosts: workers
+  become: true
+
+  vars:
+    k3s_token: "{{ hostvars[groups['control_plane'][0]]['k3s_token'] }}"
+    k3s_server_url: "https://{{ hostvars[groups['control_plane'][0]]['ansible_default_ipv4']['address'] }}:6443"
+
+  roles:
+    - k3s-agent
+
+- name: Deploy Hetzner CCM
+  hosts: control_plane[0]
+  become: true
+
+  roles:
+    - ccm
+
+- name: Finalize
+  hosts: localhost
+  connection: local
+  tasks:
+    - name: Update kubeconfig server address
+      command: |
+        sed -i 's/127.0.0.1/{{ hostvars[groups["control_plane"][0]]["ansible_default_ipv4"]["address"] }}/g' ../outputs/kubeconfig
+      changed_when: true
+
+    - name: Display success message
+      debug:
+        msg: |
+          Cluster setup complete!
+          Control planes: {{ groups['control_plane'] | length }}
+          Workers: {{ groups['workers'] | length }}
+          To access the cluster:
+            export KUBECONFIG={{ playbook_dir }}/../outputs/kubeconfig
+            kubectl get nodes