feat: implement kubeadm bootstrap scaffolding for Nix nodes

nixos/kubeadm/README.md

@@ -10,8 +10,13 @@ This folder defines role-based NixOS configs for a kubeadm cluster.
 ## What this provides
 
 - Shared Kubernetes/node prerequisites in `modules/k8s-common.nix`
+- Shared cluster defaults in `modules/k8s-cluster-settings.nix`
 - Role-specific settings for control planes and workers
-- Host configs for each node in `hosts/`
+- Bootstrap helper commands:
+  - `th-kubeadm-init`
+  - `th-kubeadm-join-control-plane`
+  - `th-kubeadm-join-worker`
+  - `th-kubeadm-status`
 
 ## Hardware config files
 
@@ -36,7 +41,56 @@ sudo nixos-rebuild switch --flake .#cp-1
 For remote target-host workflows, use your preferred deploy wrapper later
 (`nixos-rebuild --target-host ...` or deploy-rs/colmena).
 
+## Bootstrap runbook (kubeadm + kube-vip + Cilium)
+
+1. Apply Nix config on all nodes (`cp-*`, then `wk-*`).
+2. On `cp-1`, run:
+
+```bash
+sudo th-kubeadm-init
+```
+
+This infers the control-plane VIP as `<node-subnet>.250` on `eth0`, creates the
+kube-vip static pod manifest, and runs `kubeadm init`.
+
+3. Install Cilium from `cp-1`:
+
+```bash
+helm repo add cilium https://helm.cilium.io
+helm repo update
+helm upgrade --install cilium cilium/cilium \
+  --namespace kube-system \
+  --set kubeProxyReplacement=true
+```
+
+4. Generate join commands on `cp-1`:
+
+```bash
+sudo kubeadm token create --print-join-command
+sudo kubeadm init phase upload-certs --upload-certs
+```
+
+5. Join `cp-2` and `cp-3`:
+
+```bash
+sudo th-kubeadm-join-control-plane '<kubeadm join ... --control-plane --certificate-key ...>'
+```
+
+6. Join workers:
+
+```bash
+sudo th-kubeadm-join-worker '<kubeadm join ...>'
+```
+
+7. Validate from a control plane:
+
+```bash
+kubectl get nodes -o wide
+kubectl -n kube-system get pods -o wide
+```
+
 ## Notes
 
-- This does not run `kubeadm init/join` automatically.
-- It prepares OS/runtime/kernel prerequisites so kubeadm bootstrapping is clean.
+- Scripts are intentionally manual-triggered (predictable for homelab bring-up).
+- If `.250` on the node subnet is already in use, change `controlPlaneVipSuffix`
+  in `modules/k8s-cluster-settings.nix` before bootstrap.

nixos/kubeadm/hosts/cp-1.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-control-plane.nix
     ]

nixos/kubeadm/hosts/cp-2.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-control-plane.nix
     ]

nixos/kubeadm/hosts/cp-3.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-control-plane.nix
     ]

nixos/kubeadm/hosts/wk-1.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-worker.nix
     ]

nixos/kubeadm/hosts/wk-2.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-worker.nix
     ]

nixos/kubeadm/hosts/wk-3.nix

@@ -3,6 +3,7 @@
 {
   imports =
     [
+      ../modules/k8s-cluster-settings.nix
       ../modules/k8s-common.nix
       ../modules/k8s-worker.nix
     ]

nixos/kubeadm/modules/k8s-cluster-settings.nix

@@ -0,0 +1,12 @@
+{ ... }:
+
+{
+  terrahome.kubeadm = {
+    k8sMinor = "1.31";
+    controlPlaneInterface = "eth0";
+    controlPlaneVipSuffix = 250;
+    podSubnet = "10.244.0.0/16";
+    serviceSubnet = "10.96.0.0/12";
+    clusterDomain = "cluster.local";
+  };
+}

nixos/kubeadm/modules/k8s-common.nix

@@ -1,6 +1,43 @@
-{ pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
+let
+  pinnedK8s = lib.attrByPath [ "kubernetes_1_31" ] pkgs.kubernetes pkgs;
+  kubeVipImage = "ghcr.io/kube-vip/kube-vip:v0.8.9";
+in
 {
+  options.terrahome.kubeadm = {
+    k8sMinor = lib.mkOption {
+      type = lib.types.str;
+      default = "1.31";
+    };
+
+    controlPlaneInterface = lib.mkOption {
+      type = lib.types.str;
+      default = "eth0";
+    };
+
+    controlPlaneVipSuffix = lib.mkOption {
+      type = lib.types.int;
+      default = 250;
+    };
+
+    podSubnet = lib.mkOption {
+      type = lib.types.str;
+      default = "10.244.0.0/16";
+    };
+
+    serviceSubnet = lib.mkOption {
+      type = lib.types.str;
+      default = "10.96.0.0/12";
+    };
+
+    clusterDomain = lib.mkOption {
+      type = lib.types.str;
+      default = "cluster.local";
+    };
+  };
+
+  config = {
   boot.kernelModules = [ "overlay" "br_netfilter" ];
 
   boot.kernel.sysctl = {
@@ -10,6 +47,11 @@
   };
 
   virtualisation.containerd.enable = true;
+  virtualisation.containerd.settings = {
+    plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = true;
+  };
+
+  swapDevices = lib.mkForce [ ];
 
   services.openssh.enable = true;
   services.openssh.settings = {
@@ -17,19 +59,143 @@
     KbdInteractiveAuthentication = false;
   };
 
-  environment.systemPackages = with pkgs; [
+  environment.variables = {
+    KUBECONFIG = "/etc/kubernetes/admin.conf";
+    KUBE_VIP_IMAGE = kubeVipImage;
+  };
+
+  environment.systemPackages = (with pkgs; [
     containerd
     cri-tools
     cni-plugins
-    kubernetes
-    kubectl
+    pinnedK8s
     kubernetes-helm
     conntrack-tools
     socat
     ethtool
     ipvsadm
+    iproute2
+    iptables
+    ebtables
     jq
     curl
     vim
+    gawk
+  ]) ++ [
+    (pkgs.writeShellScriptBin "th-kubeadm-init" ''
+      set -euo pipefail
+
+      iface="${config.terrahome.kubeadm.controlPlaneInterface}"
+      suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}"
+      pod_subnet="${config.terrahome.kubeadm.podSubnet}"
+      service_subnet="${config.terrahome.kubeadm.serviceSubnet}"
+      domain="${config.terrahome.kubeadm.clusterDomain}"
+
+      local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}')
+      if [ -z "${local_ip_cidr:-}" ]; then
+        echo "Could not determine IPv4 CIDR on interface $iface"
+        exit 1
+      fi
+
+      subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}')
+      vip="$subnet_prefix.$suffix"
+
+      echo "Using control-plane endpoint: $vip:6443"
+      echo "Using kube-vip interface: $iface"
+
+      mkdir -p /etc/kubernetes/manifests
+      ctr image pull "$KUBE_VIP_IMAGE"
+
+      ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \
+        --interface "$iface" \
+        --address "$vip" \
+        --controlplane \
+        --services \
+        --arp \
+        --leaderElection \
+        > /etc/kubernetes/manifests/kube-vip.yaml
+
+      kubeadm init \
+        --control-plane-endpoint "$vip:6443" \
+        --upload-certs \
+        --pod-network-cidr "$pod_subnet" \
+        --service-cidr "$service_subnet" \
+        --service-dns-domain "$domain"
+
+      mkdir -p /root/.kube
+      cp /etc/kubernetes/admin.conf /root/.kube/config
+      chmod 600 /root/.kube/config
+
+      echo
+      echo "Next: install Cilium, then generate join commands:"
+      echo "  kubeadm token create --print-join-command"
+      echo "  kubeadm token create --print-join-command --certificate-key <key>"
+    '')
+
+    (pkgs.writeShellScriptBin "th-kubeadm-join-control-plane" ''
+      set -euo pipefail
+      if [ "$#" -lt 1 ]; then
+        echo "Usage: th-kubeadm-join-control-plane '<kubeadm join ... --control-plane --certificate-key ...>'"
+        exit 1
+      fi
+
+      iface="${config.terrahome.kubeadm.controlPlaneInterface}"
+      suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}"
+      local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}')
+      if [ -z "${local_ip_cidr:-}" ]; then
+        echo "Could not determine IPv4 CIDR on interface $iface"
+        exit 1
+      fi
+
+      subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}')
+      vip="$subnet_prefix.$suffix"
+
+      mkdir -p /etc/kubernetes/manifests
+      ctr image pull "$KUBE_VIP_IMAGE"
+      ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \
+        --interface "$iface" \
+        --address "$vip" \
+        --controlplane \
+        --services \
+        --arp \
+        --leaderElection \
+        > /etc/kubernetes/manifests/kube-vip.yaml
+
+      eval "$1"
+    '')
+
+    (pkgs.writeShellScriptBin "th-kubeadm-join-worker" ''
+      set -euo pipefail
+      if [ "$#" -lt 1 ]; then
+        echo "Usage: th-kubeadm-join-worker '<kubeadm join ...>'"
+        exit 1
+      fi
+
+      eval "$1"
+    '')
+
+    (pkgs.writeShellScriptBin "th-kubeadm-status" ''
+      set -euo pipefail
+      systemctl is-active containerd || true
+      systemctl is-active kubelet || true
+      crictl info >/dev/null && echo "crictl: ok" || echo "crictl: not-ready"
+    '')
   ];
+
+  systemd.services.kubelet = {
+    description = "Kubernetes Kubelet";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "containerd.service" "network-online.target" ];
+    serviceConfig = {
+      ExecStart = "${pinnedK8s}/bin/kubelet";
+      Restart = "always";
+      RestartSec = "10";
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /etc/kubernetes 0755 root root -"
+    "d /etc/kubernetes/manifests 0755 root root -"
+  ];
+  };
 }