From 7206d8cd416e769af21c83b4887bfc33c2ba3afa Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:04:14 +0000 Subject: [PATCH 1/7] feat: implement kubeadm bootstrap scaffolding for Nix nodes --- nixos/kubeadm/README.md | 60 +++++- nixos/kubeadm/hosts/cp-1.nix | 1 + nixos/kubeadm/hosts/cp-2.nix | 1 + nixos/kubeadm/hosts/cp-3.nix | 1 + nixos/kubeadm/hosts/wk-1.nix | 1 + nixos/kubeadm/hosts/wk-2.nix | 1 + nixos/kubeadm/hosts/wk-3.nix | 1 + .../kubeadm/modules/k8s-cluster-settings.nix | 12 ++ nixos/kubeadm/modules/k8s-common.nix | 174 +++++++++++++++++- 9 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 nixos/kubeadm/modules/k8s-cluster-settings.nix diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index e724000..4c3c391 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -10,8 +10,13 @@ This folder defines role-based NixOS configs for a kubeadm cluster. ## What this provides - Shared Kubernetes/node prerequisites in `modules/k8s-common.nix` +- Shared cluster defaults in `modules/k8s-cluster-settings.nix` - Role-specific settings for control planes and workers -- Host configs for each node in `hosts/` +- Bootstrap helper commands: + - `th-kubeadm-init` + - `th-kubeadm-join-control-plane` + - `th-kubeadm-join-worker` + - `th-kubeadm-status` ## Hardware config files @@ -36,7 +41,56 @@ sudo nixos-rebuild switch --flake .#cp-1 For remote target-host workflows, use your preferred deploy wrapper later (`nixos-rebuild --target-host ...` or deploy-rs/colmena). +## Bootstrap runbook (kubeadm + kube-vip + Cilium) + +1. Apply Nix config on all nodes (`cp-*`, then `wk-*`). +2. On `cp-1`, run: + +```bash +sudo th-kubeadm-init +``` + +This infers the control-plane VIP as `.250` on `eth0`, creates the +kube-vip static pod manifest, and runs `kubeadm init`. + +3. Install Cilium from `cp-1`: + +```bash +helm repo add cilium https://helm.cilium.io +helm repo update +helm upgrade --install cilium cilium/cilium \ + --namespace kube-system \ + --set kubeProxyReplacement=true +``` + +4. Generate join commands on `cp-1`: + +```bash +sudo kubeadm token create --print-join-command +sudo kubeadm init phase upload-certs --upload-certs +``` + +5. Join `cp-2` and `cp-3`: + +```bash +sudo th-kubeadm-join-control-plane '' +``` + +6. Join workers: + +```bash +sudo th-kubeadm-join-worker '' +``` + +7. Validate from a control plane: + +```bash +kubectl get nodes -o wide +kubectl -n kube-system get pods -o wide +``` + ## Notes -- This does not run `kubeadm init/join` automatically. -- It prepares OS/runtime/kernel prerequisites so kubeadm bootstrapping is clean. +- Scripts are intentionally manual-triggered (predictable for homelab bring-up). +- If `.250` on the node subnet is already in use, change `controlPlaneVipSuffix` + in `modules/k8s-cluster-settings.nix` before bootstrap. diff --git a/nixos/kubeadm/hosts/cp-1.nix b/nixos/kubeadm/hosts/cp-1.nix index 56af9ba..3de50fc 100644 --- a/nixos/kubeadm/hosts/cp-1.nix +++ b/nixos/kubeadm/hosts/cp-1.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/cp-2.nix b/nixos/kubeadm/hosts/cp-2.nix index a186ca8..03b6fb7 100644 --- a/nixos/kubeadm/hosts/cp-2.nix +++ b/nixos/kubeadm/hosts/cp-2.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/cp-3.nix b/nixos/kubeadm/hosts/cp-3.nix index 5006fff..40acfd4 100644 --- a/nixos/kubeadm/hosts/cp-3.nix +++ b/nixos/kubeadm/hosts/cp-3.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/wk-1.nix b/nixos/kubeadm/hosts/wk-1.nix index 709643e..d102e4c 100644 --- a/nixos/kubeadm/hosts/wk-1.nix +++ b/nixos/kubeadm/hosts/wk-1.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/hosts/wk-2.nix b/nixos/kubeadm/hosts/wk-2.nix index 5a11e90..78954db 100644 --- a/nixos/kubeadm/hosts/wk-2.nix +++ b/nixos/kubeadm/hosts/wk-2.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/hosts/wk-3.nix b/nixos/kubeadm/hosts/wk-3.nix index 64d11d5..1d21f36 100644 --- a/nixos/kubeadm/hosts/wk-3.nix +++ b/nixos/kubeadm/hosts/wk-3.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/modules/k8s-cluster-settings.nix b/nixos/kubeadm/modules/k8s-cluster-settings.nix new file mode 100644 index 0000000..2c89030 --- /dev/null +++ b/nixos/kubeadm/modules/k8s-cluster-settings.nix @@ -0,0 +1,12 @@ +{ ... }: + +{ + terrahome.kubeadm = { + k8sMinor = "1.31"; + controlPlaneInterface = "eth0"; + controlPlaneVipSuffix = 250; + podSubnet = "10.244.0.0/16"; + serviceSubnet = "10.96.0.0/12"; + clusterDomain = "cluster.local"; + }; +} diff --git a/nixos/kubeadm/modules/k8s-common.nix b/nixos/kubeadm/modules/k8s-common.nix index 2533868..a4f6c41 100644 --- a/nixos/kubeadm/modules/k8s-common.nix +++ b/nixos/kubeadm/modules/k8s-common.nix @@ -1,6 +1,43 @@ -{ pkgs, ... }: +{ config, lib, pkgs, ... }: +let + pinnedK8s = lib.attrByPath [ "kubernetes_1_31" ] pkgs.kubernetes pkgs; + kubeVipImage = "ghcr.io/kube-vip/kube-vip:v0.8.9"; +in { + options.terrahome.kubeadm = { + k8sMinor = lib.mkOption { + type = lib.types.str; + default = "1.31"; + }; + + controlPlaneInterface = lib.mkOption { + type = lib.types.str; + default = "eth0"; + }; + + controlPlaneVipSuffix = lib.mkOption { + type = lib.types.int; + default = 250; + }; + + podSubnet = lib.mkOption { + type = lib.types.str; + default = "10.244.0.0/16"; + }; + + serviceSubnet = lib.mkOption { + type = lib.types.str; + default = "10.96.0.0/12"; + }; + + clusterDomain = lib.mkOption { + type = lib.types.str; + default = "cluster.local"; + }; + }; + + config = { boot.kernelModules = [ "overlay" "br_netfilter" ]; boot.kernel.sysctl = { @@ -10,6 +47,11 @@ }; virtualisation.containerd.enable = true; + virtualisation.containerd.settings = { + plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = true; + }; + + swapDevices = lib.mkForce [ ]; services.openssh.enable = true; services.openssh.settings = { @@ -17,19 +59,143 @@ KbdInteractiveAuthentication = false; }; - environment.systemPackages = with pkgs; [ + environment.variables = { + KUBECONFIG = "/etc/kubernetes/admin.conf"; + KUBE_VIP_IMAGE = kubeVipImage; + }; + + environment.systemPackages = (with pkgs; [ containerd cri-tools cni-plugins - kubernetes - kubectl + pinnedK8s kubernetes-helm conntrack-tools socat ethtool ipvsadm + iproute2 + iptables + ebtables jq curl vim + gawk + ]) ++ [ + (pkgs.writeShellScriptBin "th-kubeadm-init" '' + set -euo pipefail + + iface="${config.terrahome.kubeadm.controlPlaneInterface}" + suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}" + pod_subnet="${config.terrahome.kubeadm.podSubnet}" + service_subnet="${config.terrahome.kubeadm.serviceSubnet}" + domain="${config.terrahome.kubeadm.clusterDomain}" + + local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') + if [ -z "${local_ip_cidr:-}" ]; then + echo "Could not determine IPv4 CIDR on interface $iface" + exit 1 + fi + + subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}') + vip="$subnet_prefix.$suffix" + + echo "Using control-plane endpoint: $vip:6443" + echo "Using kube-vip interface: $iface" + + mkdir -p /etc/kubernetes/manifests + ctr image pull "$KUBE_VIP_IMAGE" + + ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \ + --interface "$iface" \ + --address "$vip" \ + --controlplane \ + --services \ + --arp \ + --leaderElection \ + > /etc/kubernetes/manifests/kube-vip.yaml + + kubeadm init \ + --control-plane-endpoint "$vip:6443" \ + --upload-certs \ + --pod-network-cidr "$pod_subnet" \ + --service-cidr "$service_subnet" \ + --service-dns-domain "$domain" + + mkdir -p /root/.kube + cp /etc/kubernetes/admin.conf /root/.kube/config + chmod 600 /root/.kube/config + + echo + echo "Next: install Cilium, then generate join commands:" + echo " kubeadm token create --print-join-command" + echo " kubeadm token create --print-join-command --certificate-key " + '') + + (pkgs.writeShellScriptBin "th-kubeadm-join-control-plane" '' + set -euo pipefail + if [ "$#" -lt 1 ]; then + echo "Usage: th-kubeadm-join-control-plane ''" + exit 1 + fi + + iface="${config.terrahome.kubeadm.controlPlaneInterface}" + suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}" + local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') + if [ -z "${local_ip_cidr:-}" ]; then + echo "Could not determine IPv4 CIDR on interface $iface" + exit 1 + fi + + subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}') + vip="$subnet_prefix.$suffix" + + mkdir -p /etc/kubernetes/manifests + ctr image pull "$KUBE_VIP_IMAGE" + ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \ + --interface "$iface" \ + --address "$vip" \ + --controlplane \ + --services \ + --arp \ + --leaderElection \ + > /etc/kubernetes/manifests/kube-vip.yaml + + eval "$1" + '') + + (pkgs.writeShellScriptBin "th-kubeadm-join-worker" '' + set -euo pipefail + if [ "$#" -lt 1 ]; then + echo "Usage: th-kubeadm-join-worker ''" + exit 1 + fi + + eval "$1" + '') + + (pkgs.writeShellScriptBin "th-kubeadm-status" '' + set -euo pipefail + systemctl is-active containerd || true + systemctl is-active kubelet || true + crictl info >/dev/null && echo "crictl: ok" || echo "crictl: not-ready" + '') ]; + + systemd.services.kubelet = { + description = "Kubernetes Kubelet"; + wantedBy = [ "multi-user.target" ]; + after = [ "containerd.service" "network-online.target" ]; + serviceConfig = { + ExecStart = "${pinnedK8s}/bin/kubelet"; + Restart = "always"; + RestartSec = "10"; + }; + }; + + systemd.tmpfiles.rules = [ + "d /etc/kubernetes 0755 root root -" + "d /etc/kubernetes/manifests 0755 root root -" + ]; + }; } From abac6300ca70ca560aed4e44fb36192c98982897 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:09:05 +0000 Subject: [PATCH 2/7] refactor: generate kubeadm host configs from flake --- nixos/kubeadm/README.md | 3 ++- nixos/kubeadm/flake.nix | 32 ++++++++++++++++++++++---------- nixos/kubeadm/hosts/cp-1.nix | 15 --------------- nixos/kubeadm/hosts/cp-2.nix | 15 --------------- nixos/kubeadm/hosts/cp-3.nix | 15 --------------- nixos/kubeadm/hosts/wk-1.nix | 15 --------------- nixos/kubeadm/hosts/wk-2.nix | 15 --------------- nixos/kubeadm/hosts/wk-3.nix | 15 --------------- 8 files changed, 24 insertions(+), 101 deletions(-) delete mode 100644 nixos/kubeadm/hosts/cp-1.nix delete mode 100644 nixos/kubeadm/hosts/cp-2.nix delete mode 100644 nixos/kubeadm/hosts/cp-3.nix delete mode 100644 nixos/kubeadm/hosts/wk-1.nix delete mode 100644 nixos/kubeadm/hosts/wk-2.nix delete mode 100644 nixos/kubeadm/hosts/wk-3.nix diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index 4c3c391..38f5303 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -12,6 +12,7 @@ This folder defines role-based NixOS configs for a kubeadm cluster. - Shared Kubernetes/node prerequisites in `modules/k8s-common.nix` - Shared cluster defaults in `modules/k8s-cluster-settings.nix` - Role-specific settings for control planes and workers +- Generated per-node host configs from `flake.nix` (no duplicated host files) - Bootstrap helper commands: - `th-kubeadm-init` - `th-kubeadm-join-control-plane` @@ -20,7 +21,7 @@ This folder defines role-based NixOS configs for a kubeadm cluster. ## Hardware config files -Each host file optionally imports `hosts/hardware/.nix` if present. +The flake automatically imports `hosts/hardware/.nix` if present. Copy each node's generated hardware config into this folder: ```bash diff --git a/nixos/kubeadm/flake.nix b/nixos/kubeadm/flake.nix index d3dacef..eca29ac 100644 --- a/nixos/kubeadm/flake.nix +++ b/nixos/kubeadm/flake.nix @@ -8,19 +8,31 @@ outputs = { nixpkgs, ... }: let system = "x86_64-linux"; - mkHost = hostModules: + lib = nixpkgs.lib; + + mkNode = { name, role }: + let + roleModule = if role == "control-plane" then ./modules/k8s-control-plane.nix else ./modules/k8s-worker.nix; + hardwarePath = ./hosts/hardware + "/${name}.nix"; + in nixpkgs.lib.nixosSystem { inherit system; - modules = hostModules; + modules = [ + ./modules/k8s-cluster-settings.nix + ./modules/k8s-common.nix + roleModule + ({ lib, ... }: { + imports = lib.optional (builtins.pathExists hardwarePath) hardwarePath; + networking.hostName = name; + system.stateVersion = "25.05"; + }) + ]; }; in { - nixosConfigurations = { - cp-1 = mkHost [ ./hosts/cp-1.nix ]; - cp-2 = mkHost [ ./hosts/cp-2.nix ]; - cp-3 = mkHost [ ./hosts/cp-3.nix ]; - wk-1 = mkHost [ ./hosts/wk-1.nix ]; - wk-2 = mkHost [ ./hosts/wk-2.nix ]; - wk-3 = mkHost [ ./hosts/wk-3.nix ]; - }; + nixosConfigurations = lib.genAttrs [ "cp-1" "cp-2" "cp-3" "wk-1" "wk-2" "wk-3" ] (name: + mkNode { + inherit name; + role = if lib.hasPrefix "cp-" name then "control-plane" else "worker"; + }); }; } diff --git a/nixos/kubeadm/hosts/cp-1.nix b/nixos/kubeadm/hosts/cp-1.nix deleted file mode 100644 index 3de50fc..0000000 --- a/nixos/kubeadm/hosts/cp-1.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-control-plane.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/cp-1.nix) ./hardware/cp-1.nix; - - networking.hostName = "cp-1"; - - system.stateVersion = "25.05"; -} diff --git a/nixos/kubeadm/hosts/cp-2.nix b/nixos/kubeadm/hosts/cp-2.nix deleted file mode 100644 index 03b6fb7..0000000 --- a/nixos/kubeadm/hosts/cp-2.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-control-plane.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/cp-2.nix) ./hardware/cp-2.nix; - - networking.hostName = "cp-2"; - - system.stateVersion = "25.05"; -} diff --git a/nixos/kubeadm/hosts/cp-3.nix b/nixos/kubeadm/hosts/cp-3.nix deleted file mode 100644 index 40acfd4..0000000 --- a/nixos/kubeadm/hosts/cp-3.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-control-plane.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/cp-3.nix) ./hardware/cp-3.nix; - - networking.hostName = "cp-3"; - - system.stateVersion = "25.05"; -} diff --git a/nixos/kubeadm/hosts/wk-1.nix b/nixos/kubeadm/hosts/wk-1.nix deleted file mode 100644 index d102e4c..0000000 --- a/nixos/kubeadm/hosts/wk-1.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-worker.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/wk-1.nix) ./hardware/wk-1.nix; - - networking.hostName = "wk-1"; - - system.stateVersion = "25.05"; -} diff --git a/nixos/kubeadm/hosts/wk-2.nix b/nixos/kubeadm/hosts/wk-2.nix deleted file mode 100644 index 78954db..0000000 --- a/nixos/kubeadm/hosts/wk-2.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-worker.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/wk-2.nix) ./hardware/wk-2.nix; - - networking.hostName = "wk-2"; - - system.stateVersion = "25.05"; -} diff --git a/nixos/kubeadm/hosts/wk-3.nix b/nixos/kubeadm/hosts/wk-3.nix deleted file mode 100644 index 1d21f36..0000000 --- a/nixos/kubeadm/hosts/wk-3.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib, ... }: - -{ - imports = - [ - ../modules/k8s-cluster-settings.nix - ../modules/k8s-common.nix - ../modules/k8s-worker.nix - ] - ++ lib.optional (builtins.pathExists ./hardware/wk-3.nix) ./hardware/wk-3.nix; - - networking.hostName = "wk-3"; - - system.stateVersion = "25.05"; -} From 91dd20e60ea13faadfc48c3b24fbf2cfc9eedd16 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:12:25 +0000 Subject: [PATCH 3/7] fix: escape shell expansion in kubeadm helper scripts --- nixos/kubeadm/flake.lock | 27 +++++++++++++++++++++++++++ nixos/kubeadm/modules/k8s-common.nix | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 nixos/kubeadm/flake.lock diff --git a/nixos/kubeadm/flake.lock b/nixos/kubeadm/flake.lock new file mode 100644 index 0000000..f428ce8 --- /dev/null +++ b/nixos/kubeadm/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1767313136, + "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nixos/kubeadm/modules/k8s-common.nix b/nixos/kubeadm/modules/k8s-common.nix index a4f6c41..9b80c01 100644 --- a/nixos/kubeadm/modules/k8s-common.nix +++ b/nixos/kubeadm/modules/k8s-common.nix @@ -92,7 +92,7 @@ in domain="${config.terrahome.kubeadm.clusterDomain}" local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') - if [ -z "${local_ip_cidr:-}" ]; then + if [ -z "''${local_ip_cidr:-}" ]; then echo "Could not determine IPv4 CIDR on interface $iface" exit 1 fi @@ -142,7 +142,7 @@ in iface="${config.terrahome.kubeadm.controlPlaneInterface}" suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}" local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') - if [ -z "${local_ip_cidr:-}" ]; then + if [ -z "''${local_ip_cidr:-}" ]; then echo "Could not determine IPv4 CIDR on interface $iface" exit 1 fi From 885a92f4940cd0018f8a8cc325018f9416407c56 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:19:37 +0000 Subject: [PATCH 4/7] chore: add lightweight flake checks for kubeadm configs --- nixos/kubeadm/flake.nix | 49 +++++++++++++++++++++++++--- nixos/kubeadm/modules/k8s-common.nix | 1 + 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/nixos/kubeadm/flake.nix b/nixos/kubeadm/flake.nix index eca29ac..e88ee5c 100644 --- a/nixos/kubeadm/flake.nix +++ b/nixos/kubeadm/flake.nix @@ -9,8 +9,14 @@ let system = "x86_64-linux"; lib = nixpkgs.lib; + pkgs = nixpkgs.legacyPackages.${system}; + nodeNames = [ "cp-1" "cp-2" "cp-3" "wk-1" "wk-2" "wk-3" ]; - mkNode = { name, role }: + mkNode = { + name, + role, + extraModules ? [ ], + }: let roleModule = if role == "control-plane" then ./modules/k8s-control-plane.nix else ./modules/k8s-worker.nix; hardwarePath = ./hosts/hardware + "/${name}.nix"; @@ -25,14 +31,47 @@ imports = lib.optional (builtins.pathExists hardwarePath) hardwarePath; networking.hostName = name; system.stateVersion = "25.05"; + boot.loader.grub.devices = lib.mkDefault [ "/dev/sda" ]; + fileSystems."/" = lib.mkDefault { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; }) - ]; + ] ++ extraModules; }; - in { - nixosConfigurations = lib.genAttrs [ "cp-1" "cp-2" "cp-3" "wk-1" "wk-2" "wk-3" ] (name: + + mkNodeByName = name: mkNode { inherit name; role = if lib.hasPrefix "cp-" name then "control-plane" else "worker"; - }); + }; + + mkEvalCheck = name: + let + cfg = mkNode { + inherit name; + role = if lib.hasPrefix "cp-" name then "control-plane" else "worker"; + extraModules = [ + ({ lib, ... }: { + boot.loader.grub.devices = lib.mkDefault [ "/dev/sda" ]; + fileSystems."/" = lib.mkDefault { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; + }) + ]; + }; + in + pkgs.runCommand "eval-${name}" { } '' + cat > "$out" <<'EOF' + host=${cfg.config.networking.hostName} + role=${if lib.hasPrefix "cp-" name then "control-plane" else "worker"} + stateVersion=${cfg.config.system.stateVersion} + EOF + ''; + in { + nixosConfigurations = lib.genAttrs nodeNames mkNodeByName; + + checks.${system} = lib.genAttrs nodeNames mkEvalCheck; }; } diff --git a/nixos/kubeadm/modules/k8s-common.nix b/nixos/kubeadm/modules/k8s-common.nix index 9b80c01..57fea49 100644 --- a/nixos/kubeadm/modules/k8s-common.nix +++ b/nixos/kubeadm/modules/k8s-common.nix @@ -185,6 +185,7 @@ in systemd.services.kubelet = { description = "Kubernetes Kubelet"; wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; after = [ "containerd.service" "network-online.target" ]; serviceConfig = { ExecStart = "${pinnedK8s}/bin/kubelet"; From 9fe845b53d936b289b17aadafa2722334c23e7f2 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:24:45 +0000 Subject: [PATCH 5/7] feat: add repeatable kubeadm rebuild and reset scripts --- nixos/kubeadm/README.md | 25 ++++++ nixos/kubeadm/scripts/inventory.example.env | 11 +++ .../kubeadm/scripts/rebuild-and-bootstrap.sh | 89 +++++++++++++++++++ nixos/kubeadm/scripts/reset-cluster-nodes.sh | 37 ++++++++ 4 files changed, 162 insertions(+) create mode 100644 nixos/kubeadm/scripts/inventory.example.env create mode 100755 nixos/kubeadm/scripts/rebuild-and-bootstrap.sh create mode 100755 nixos/kubeadm/scripts/reset-cluster-nodes.sh diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index 38f5303..794bf82 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -90,6 +90,31 @@ kubectl get nodes -o wide kubectl -n kube-system get pods -o wide ``` +## Repeatable rebuild flow (recommended) + +1. Copy and edit inventory: + +```bash +cp ./scripts/inventory.example.env ./scripts/inventory.env +$EDITOR ./scripts/inventory.env +``` + +2. Rebuild all nodes and bootstrap cluster: + +```bash +./scripts/rebuild-and-bootstrap.sh +``` + +3. If you only want to reset Kubernetes state on existing VMs: + +```bash +./scripts/reset-cluster-nodes.sh +``` + +For a full nuke/recreate lifecycle: +- run Terraform destroy/apply for VMs first, +- then run `./scripts/rebuild-and-bootstrap.sh` again. + ## Notes - Scripts are intentionally manual-triggered (predictable for homelab bring-up). diff --git a/nixos/kubeadm/scripts/inventory.example.env b/nixos/kubeadm/scripts/inventory.example.env new file mode 100644 index 0000000..9501588 --- /dev/null +++ b/nixos/kubeadm/scripts/inventory.example.env @@ -0,0 +1,11 @@ +SSH_USER=micqdf + +# Control planes +CP_1=192.168.1.101 +CP_2=192.168.1.102 +CP_3=192.168.1.103 + +# Workers +WK_1=192.168.1.111 +WK_2=192.168.1.112 +WK_3=192.168.1.113 diff --git a/nixos/kubeadm/scripts/rebuild-and-bootstrap.sh b/nixos/kubeadm/scripts/rebuild-and-bootstrap.sh new file mode 100755 index 0000000..d38c2bc --- /dev/null +++ b/nixos/kubeadm/scripts/rebuild-and-bootstrap.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FLAKE_DIR="${FLAKE_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}" +INVENTORY_FILE="${1:-$SCRIPT_DIR/inventory.env}" + +if [ ! -f "$INVENTORY_FILE" ]; then + echo "Missing inventory file: $INVENTORY_FILE" + echo "Copy $SCRIPT_DIR/inventory.example.env to $SCRIPT_DIR/inventory.env and edit IPs." + exit 1 +fi + +# shellcheck disable=SC1090 +source "$INVENTORY_FILE" + +SSH_USER="${SSH_USER:-micqdf}" +SSH_OPTS="${SSH_OPTS:- -o BatchMode=yes -o StrictHostKeyChecking=accept-new }" + +required=(CP_1 CP_2 CP_3 WK_1 WK_2 WK_3) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "Missing required inventory variable: $key" + exit 1 + fi +done + +remote() { + local host_ip="$1" + local cmd="$2" + ssh $SSH_OPTS "$SSH_USER@$host_ip" "$cmd" +} + +rebuild_node() { + local node_name="$1" + local node_ip="$2" + + echo "==> Rebuilding $node_name on $node_ip" + nixos-rebuild switch \ + --flake "$FLAKE_DIR#$node_name" \ + --target-host "$SSH_USER@$node_ip" \ + --use-remote-sudo +} + +for node in cp-1 cp-2 cp-3 wk-1 wk-2 wk-3; do + key="${node^^}" + key="${key//-/_}" + rebuild_node "$node" "${!key}" +done + +echo "==> Initializing control plane on cp-1" +remote "$CP_1" "sudo th-kubeadm-init" + +echo "==> Installing Cilium on cp-1" +remote "$CP_1" "helm repo add cilium https://helm.cilium.io >/dev/null 2>&1 || true" +remote "$CP_1" "helm repo update >/dev/null" +remote "$CP_1" "kubectl create namespace kube-system >/dev/null 2>&1 || true" +remote "$CP_1" "helm upgrade --install cilium cilium/cilium --namespace kube-system --set kubeProxyReplacement=true" + +echo "==> Building kubeadm join commands" +JOIN_CMD="$(remote "$CP_1" "sudo kubeadm token create --print-join-command")" +CERT_KEY="$(remote "$CP_1" "sudo kubeadm init phase upload-certs --upload-certs | tail -n 1")" +CP_JOIN_CMD="$JOIN_CMD --control-plane --certificate-key $CERT_KEY" + +join_control_plane() { + local node_ip="$1" + local encoded + encoded="$(printf '%s' "$CP_JOIN_CMD" | base64 -w0)" + remote "$node_ip" "sudo th-kubeadm-join-control-plane \"\$(echo $encoded | base64 -d)\"" +} + +join_worker() { + local node_ip="$1" + local encoded + encoded="$(printf '%s' "$JOIN_CMD" | base64 -w0)" + remote "$node_ip" "sudo th-kubeadm-join-worker \"\$(echo $encoded | base64 -d)\"" +} + +echo "==> Joining remaining control planes" +join_control_plane "$CP_2" +join_control_plane "$CP_3" + +echo "==> Joining workers" +join_worker "$WK_1" +join_worker "$WK_2" +join_worker "$WK_3" + +echo "==> Final node list" +remote "$CP_1" "kubectl get nodes -o wide" diff --git a/nixos/kubeadm/scripts/reset-cluster-nodes.sh b/nixos/kubeadm/scripts/reset-cluster-nodes.sh new file mode 100755 index 0000000..733c635 --- /dev/null +++ b/nixos/kubeadm/scripts/reset-cluster-nodes.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INVENTORY_FILE="${1:-$SCRIPT_DIR/inventory.env}" + +if [ ! -f "$INVENTORY_FILE" ]; then + echo "Missing inventory file: $INVENTORY_FILE" + echo "Copy $SCRIPT_DIR/inventory.example.env to $SCRIPT_DIR/inventory.env and edit IPs." + exit 1 +fi + +# shellcheck disable=SC1090 +source "$INVENTORY_FILE" + +SSH_USER="${SSH_USER:-micqdf}" +SSH_OPTS="${SSH_OPTS:- -o BatchMode=yes -o StrictHostKeyChecking=accept-new }" + +required=(CP_1 CP_2 CP_3 WK_1 WK_2 WK_3) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "Missing required inventory variable: $key" + exit 1 + fi +done + +reset_node() { + local node_ip="$1" + echo "==> Resetting $node_ip" + ssh $SSH_OPTS "$SSH_USER@$node_ip" "sudo kubeadm reset -f && sudo systemctl stop kubelet && sudo rm -rf /etc/kubernetes /var/lib/etcd /var/lib/cni /etc/cni/net.d" +} + +for key in CP_1 CP_2 CP_3 WK_1 WK_2 WK_3; do + reset_node "${!key}" +done + +echo "Cluster components reset on all listed nodes." From b0779c51c0b22a34847a2b04ebea3eef8d9ae2fb Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:26:51 +0000 Subject: [PATCH 6/7] feat: add gitea workflows for kubeadm bootstrap and reset --- .gitea/workflows/kubeadm-bootstrap.yml | 64 ++++++++++++++++++++++++++ .gitea/workflows/kubeadm-reset.yml | 55 ++++++++++++++++++++++ nixos/kubeadm/README.md | 14 ++++++ 3 files changed, 133 insertions(+) create mode 100644 .gitea/workflows/kubeadm-bootstrap.yml create mode 100644 .gitea/workflows/kubeadm-reset.yml diff --git a/.gitea/workflows/kubeadm-bootstrap.yml b/.gitea/workflows/kubeadm-bootstrap.yml new file mode 100644 index 0000000..6e1b06a --- /dev/null +++ b/.gitea/workflows/kubeadm-bootstrap.yml @@ -0,0 +1,64 @@ +name: Kubeadm Bootstrap +run-name: ${{ gitea.actor }} requested kubeadm bootstrap + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type BOOTSTRAP to run rebuild + kubeadm bootstrap" + required: true + type: string + +concurrency: + group: kubeadm-bootstrap + cancel-in-progress: false + +jobs: + bootstrap: + name: "Rebuild and Bootstrap Cluster" + runs-on: ubuntu-latest + + steps: + - name: Validate confirmation phrase + run: | + if [ "${{ inputs.confirm }}" != "BOOTSTRAP" ]; then + echo "Confirmation failed. You must type BOOTSTRAP." + exit 1 + fi + + - name: Checkout repository + uses: https://gitea.com/actions/checkout@v4 + + - name: Create SSH key + run: | + install -m 0700 -d ~/.ssh + cat > ~/.ssh/id_ed25519 << 'EOF' + ${{ secrets.KUBEADM_SSH_PRIVATE_KEY }} + EOF + chmod 0600 ~/.ssh/id_ed25519 + + - name: Create kubeadm inventory + working-directory: nixos/kubeadm/scripts + run: | + cat > inventory.env << EOF + SSH_USER=${{ secrets.KUBEADM_SSH_USER }} + CP_1=${{ secrets.KUBEADM_CP_1_IP }} + CP_2=${{ secrets.KUBEADM_CP_2_IP }} + CP_3=${{ secrets.KUBEADM_CP_3_IP }} + WK_1=${{ secrets.KUBEADM_WK_1_IP }} + WK_2=${{ secrets.KUBEADM_WK_2_IP }} + WK_3=${{ secrets.KUBEADM_WK_3_IP }} + EOF + + - name: Validate nix installation + run: | + if [ ! -x /nix/var/nix/profiles/default/bin/nix ]; then + echo "Nix not found at /nix/var/nix/profiles/default/bin/nix" + exit 1 + fi + + - name: Run cluster rebuild and bootstrap + env: + PATH: /nix/var/nix/profiles/default/bin:${{ env.PATH }} + run: | + ./nixos/kubeadm/scripts/rebuild-and-bootstrap.sh diff --git a/.gitea/workflows/kubeadm-reset.yml b/.gitea/workflows/kubeadm-reset.yml new file mode 100644 index 0000000..998b2e6 --- /dev/null +++ b/.gitea/workflows/kubeadm-reset.yml @@ -0,0 +1,55 @@ +name: Kubeadm Reset +run-name: ${{ gitea.actor }} requested kubeadm reset + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type RESET to run kubeadm reset on all nodes" + required: true + type: string + +concurrency: + group: kubeadm-bootstrap + cancel-in-progress: false + +jobs: + reset: + name: "Reset Cluster Nodes" + runs-on: ubuntu-latest + + steps: + - name: Validate confirmation phrase + run: | + if [ "${{ inputs.confirm }}" != "RESET" ]; then + echo "Confirmation failed. You must type RESET." + exit 1 + fi + + - name: Checkout repository + uses: https://gitea.com/actions/checkout@v4 + + - name: Create SSH key + run: | + install -m 0700 -d ~/.ssh + cat > ~/.ssh/id_ed25519 << 'EOF' + ${{ secrets.KUBEADM_SSH_PRIVATE_KEY }} + EOF + chmod 0600 ~/.ssh/id_ed25519 + + - name: Create kubeadm inventory + working-directory: nixos/kubeadm/scripts + run: | + cat > inventory.env << EOF + SSH_USER=${{ secrets.KUBEADM_SSH_USER }} + CP_1=${{ secrets.KUBEADM_CP_1_IP }} + CP_2=${{ secrets.KUBEADM_CP_2_IP }} + CP_3=${{ secrets.KUBEADM_CP_3_IP }} + WK_1=${{ secrets.KUBEADM_WK_1_IP }} + WK_2=${{ secrets.KUBEADM_WK_2_IP }} + WK_3=${{ secrets.KUBEADM_WK_3_IP }} + EOF + + - name: Run cluster reset + run: | + ./nixos/kubeadm/scripts/reset-cluster-nodes.sh diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index 794bf82..0b2a4d1 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -115,6 +115,20 @@ For a full nuke/recreate lifecycle: - run Terraform destroy/apply for VMs first, - then run `./scripts/rebuild-and-bootstrap.sh` again. +## Optional Gitea workflow automation + +Manual dispatch workflows are available: + +- `.gitea/workflows/kubeadm-bootstrap.yml` +- `.gitea/workflows/kubeadm-reset.yml` + +Required repository secrets: + +- `KUBEADM_SSH_PRIVATE_KEY` +- `KUBEADM_SSH_USER` +- `KUBEADM_CP_1_IP`, `KUBEADM_CP_2_IP`, `KUBEADM_CP_3_IP` +- `KUBEADM_WK_1_IP`, `KUBEADM_WK_2_IP`, `KUBEADM_WK_3_IP` + ## Notes - Scripts are intentionally manual-triggered (predictable for homelab bring-up). From 8bcc162956581e673aa40a8bf4ce3a5a3a7559af Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 16:31:23 +0000 Subject: [PATCH 7/7] feat: auto-discover kubeadm node IPs from terraform state --- .gitea/workflows/kubeadm-bootstrap.yml | 81 +++++++++++++++++++++----- .gitea/workflows/kubeadm-reset.yml | 81 +++++++++++++++++++++----- nixos/kubeadm/README.md | 12 ++-- terraform/outputs.tf | 14 +++++ 4 files changed, 158 insertions(+), 30 deletions(-) diff --git a/.gitea/workflows/kubeadm-bootstrap.yml b/.gitea/workflows/kubeadm-bootstrap.yml index 6e1b06a..7da5ae2 100644 --- a/.gitea/workflows/kubeadm-bootstrap.yml +++ b/.gitea/workflows/kubeadm-bootstrap.yml @@ -32,22 +32,77 @@ jobs: - name: Create SSH key run: | install -m 0700 -d ~/.ssh - cat > ~/.ssh/id_ed25519 << 'EOF' - ${{ secrets.KUBEADM_SSH_PRIVATE_KEY }} - EOF + KEY_CONTENT="$(printf '%s' "${{ secrets.KUBEADM_SSH_PRIVATE_KEY }}")" + if [ -z "$KEY_CONTENT" ]; then + KEY_CONTENT="$(printf '%s' "${{ secrets.SSH_KEY_PRIVATE }}")" + fi + + if [ -z "$KEY_CONTENT" ]; then + echo "Missing SSH private key secret. Set KUBEADM_SSH_PRIVATE_KEY or SSH_KEY_PRIVATE." + exit 1 + fi + + printf '%s\n' "$KEY_CONTENT" > ~/.ssh/id_ed25519 chmod 0600 ~/.ssh/id_ed25519 - - name: Create kubeadm inventory - working-directory: nixos/kubeadm/scripts + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.6.6 + terraform_wrapper: false + + - name: Build Terraform backend files + working-directory: terraform run: | - cat > inventory.env << EOF - SSH_USER=${{ secrets.KUBEADM_SSH_USER }} - CP_1=${{ secrets.KUBEADM_CP_1_IP }} - CP_2=${{ secrets.KUBEADM_CP_2_IP }} - CP_3=${{ secrets.KUBEADM_CP_3_IP }} - WK_1=${{ secrets.KUBEADM_WK_1_IP }} - WK_2=${{ secrets.KUBEADM_WK_2_IP }} - WK_3=${{ secrets.KUBEADM_WK_3_IP }} + cat > secrets.auto.tfvars << EOF + pm_api_token_secret = "${{ secrets.PM_API_TOKEN_SECRET }}" + SSH_KEY_PUBLIC = "$(printf '%s' "${{ secrets.SSH_KEY_PUBLIC }}" | tr -d '\r\n')" + EOF + + cat > backend.hcl << EOF + bucket = "${{ secrets.B2_TF_BUCKET }}" + key = "terraform.tfstate" + region = "us-east-005" + endpoints = { + s3 = "${{ secrets.B2_TF_ENDPOINT }}" + } + access_key = "$(printf '%s' "${{ secrets.B2_KEY_ID }}" | tr -d '\r\n')" + secret_key = "$(printf '%s' "${{ secrets.B2_APPLICATION_KEY }}" | tr -d '\r\n')" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_path_style = true + EOF + + - name: Terraform init for state read + working-directory: terraform + run: terraform init -reconfigure -backend-config=backend.hcl + + - name: Create kubeadm inventory + run: | + TF_OUTPUT_JSON="$(terraform -chdir=terraform output -json)" + + CP_1="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-1"])' <<< "$TF_OUTPUT_JSON")" + CP_2="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-2"])' <<< "$TF_OUTPUT_JSON")" + CP_3="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-3"])' <<< "$TF_OUTPUT_JSON")" + WK_1="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-1"])' <<< "$TF_OUTPUT_JSON")" + WK_2="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-2"])' <<< "$TF_OUTPUT_JSON")" + WK_3="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-3"])' <<< "$TF_OUTPUT_JSON")" + + SSH_USER="$(printf '%s' "${{ secrets.KUBEADM_SSH_USER }}")" + if [ -z "$SSH_USER" ]; then + SSH_USER="micqdf" + fi + + cat > nixos/kubeadm/scripts/inventory.env << EOF + SSH_USER=$SSH_USER + CP_1=$CP_1 + CP_2=$CP_2 + CP_3=$CP_3 + WK_1=$WK_1 + WK_2=$WK_2 + WK_3=$WK_3 EOF - name: Validate nix installation diff --git a/.gitea/workflows/kubeadm-reset.yml b/.gitea/workflows/kubeadm-reset.yml index 998b2e6..a7b04c5 100644 --- a/.gitea/workflows/kubeadm-reset.yml +++ b/.gitea/workflows/kubeadm-reset.yml @@ -32,22 +32,77 @@ jobs: - name: Create SSH key run: | install -m 0700 -d ~/.ssh - cat > ~/.ssh/id_ed25519 << 'EOF' - ${{ secrets.KUBEADM_SSH_PRIVATE_KEY }} - EOF + KEY_CONTENT="$(printf '%s' "${{ secrets.KUBEADM_SSH_PRIVATE_KEY }}")" + if [ -z "$KEY_CONTENT" ]; then + KEY_CONTENT="$(printf '%s' "${{ secrets.SSH_KEY_PRIVATE }}")" + fi + + if [ -z "$KEY_CONTENT" ]; then + echo "Missing SSH private key secret. Set KUBEADM_SSH_PRIVATE_KEY or SSH_KEY_PRIVATE." + exit 1 + fi + + printf '%s\n' "$KEY_CONTENT" > ~/.ssh/id_ed25519 chmod 0600 ~/.ssh/id_ed25519 - - name: Create kubeadm inventory - working-directory: nixos/kubeadm/scripts + - name: Set up Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.6.6 + terraform_wrapper: false + + - name: Build Terraform backend files + working-directory: terraform run: | - cat > inventory.env << EOF - SSH_USER=${{ secrets.KUBEADM_SSH_USER }} - CP_1=${{ secrets.KUBEADM_CP_1_IP }} - CP_2=${{ secrets.KUBEADM_CP_2_IP }} - CP_3=${{ secrets.KUBEADM_CP_3_IP }} - WK_1=${{ secrets.KUBEADM_WK_1_IP }} - WK_2=${{ secrets.KUBEADM_WK_2_IP }} - WK_3=${{ secrets.KUBEADM_WK_3_IP }} + cat > secrets.auto.tfvars << EOF + pm_api_token_secret = "${{ secrets.PM_API_TOKEN_SECRET }}" + SSH_KEY_PUBLIC = "$(printf '%s' "${{ secrets.SSH_KEY_PUBLIC }}" | tr -d '\r\n')" + EOF + + cat > backend.hcl << EOF + bucket = "${{ secrets.B2_TF_BUCKET }}" + key = "terraform.tfstate" + region = "us-east-005" + endpoints = { + s3 = "${{ secrets.B2_TF_ENDPOINT }}" + } + access_key = "$(printf '%s' "${{ secrets.B2_KEY_ID }}" | tr -d '\r\n')" + secret_key = "$(printf '%s' "${{ secrets.B2_APPLICATION_KEY }}" | tr -d '\r\n')" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_path_style = true + EOF + + - name: Terraform init for state read + working-directory: terraform + run: terraform init -reconfigure -backend-config=backend.hcl + + - name: Create kubeadm inventory + run: | + TF_OUTPUT_JSON="$(terraform -chdir=terraform output -json)" + + CP_1="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-1"])' <<< "$TF_OUTPUT_JSON")" + CP_2="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-2"])' <<< "$TF_OUTPUT_JSON")" + CP_3="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["control_plane_vm_ipv4"]["value"]["cp-3"])' <<< "$TF_OUTPUT_JSON")" + WK_1="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-1"])' <<< "$TF_OUTPUT_JSON")" + WK_2="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-2"])' <<< "$TF_OUTPUT_JSON")" + WK_3="$(python3 -c 'import json,sys; d=json.loads(sys.stdin.read()); print(d["worker_vm_ipv4"]["value"]["wk-3"])' <<< "$TF_OUTPUT_JSON")" + + SSH_USER="$(printf '%s' "${{ secrets.KUBEADM_SSH_USER }}")" + if [ -z "$SSH_USER" ]; then + SSH_USER="micqdf" + fi + + cat > nixos/kubeadm/scripts/inventory.env << EOF + SSH_USER=$SSH_USER + CP_1=$CP_1 + CP_2=$CP_2 + CP_3=$CP_3 + WK_1=$WK_1 + WK_2=$WK_2 + WK_3=$WK_3 EOF - name: Run cluster reset diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index 0b2a4d1..30e2873 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -124,10 +124,14 @@ Manual dispatch workflows are available: Required repository secrets: -- `KUBEADM_SSH_PRIVATE_KEY` -- `KUBEADM_SSH_USER` -- `KUBEADM_CP_1_IP`, `KUBEADM_CP_2_IP`, `KUBEADM_CP_3_IP` -- `KUBEADM_WK_1_IP`, `KUBEADM_WK_2_IP`, `KUBEADM_WK_3_IP` +- Existing Terraform/backend secrets used by current workflows (`B2_*`, `PM_API_TOKEN_SECRET`, `SSH_KEY_PUBLIC`) +- SSH private key: prefer `KUBEADM_SSH_PRIVATE_KEY`, fallback to existing `SSH_KEY_PRIVATE` + +Optional secrets: + +- `KUBEADM_SSH_USER` (defaults to `micqdf`) + +Node IPs are auto-discovered from Terraform state outputs (`control_plane_vm_ipv4`, `worker_vm_ipv4`), so you do not need per-node IP secrets. ## Notes diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 8ec628f..c817350 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -9,6 +9,13 @@ output "control_plane_vm_names" { value = [for vm in proxmox_vm_qemu.control_planes : vm.name] } +output "control_plane_vm_ipv4" { + value = { + for vm in proxmox_vm_qemu.control_planes : + vm.name => vm.default_ipv4_address + } +} + output "worker_vm_ids" { value = { for i in range(var.worker_count) : @@ -19,3 +26,10 @@ output "worker_vm_ids" { output "worker_vm_names" { value = [for vm in proxmox_vm_qemu.workers : vm.name] } + +output "worker_vm_ipv4" { + value = { + for vm in proxmox_vm_qemu.workers : + vm.name => vm.default_ipv4_address + } +}