diff --git a/nixos/kubeadm/README.md b/nixos/kubeadm/README.md index e724000..4c3c391 100644 --- a/nixos/kubeadm/README.md +++ b/nixos/kubeadm/README.md @@ -10,8 +10,13 @@ This folder defines role-based NixOS configs for a kubeadm cluster. ## What this provides - Shared Kubernetes/node prerequisites in `modules/k8s-common.nix` +- Shared cluster defaults in `modules/k8s-cluster-settings.nix` - Role-specific settings for control planes and workers -- Host configs for each node in `hosts/` +- Bootstrap helper commands: + - `th-kubeadm-init` + - `th-kubeadm-join-control-plane` + - `th-kubeadm-join-worker` + - `th-kubeadm-status` ## Hardware config files @@ -36,7 +41,56 @@ sudo nixos-rebuild switch --flake .#cp-1 For remote target-host workflows, use your preferred deploy wrapper later (`nixos-rebuild --target-host ...` or deploy-rs/colmena). +## Bootstrap runbook (kubeadm + kube-vip + Cilium) + +1. Apply Nix config on all nodes (`cp-*`, then `wk-*`). +2. On `cp-1`, run: + +```bash +sudo th-kubeadm-init +``` + +This infers the control-plane VIP as `.250` on `eth0`, creates the +kube-vip static pod manifest, and runs `kubeadm init`. + +3. Install Cilium from `cp-1`: + +```bash +helm repo add cilium https://helm.cilium.io +helm repo update +helm upgrade --install cilium cilium/cilium \ + --namespace kube-system \ + --set kubeProxyReplacement=true +``` + +4. Generate join commands on `cp-1`: + +```bash +sudo kubeadm token create --print-join-command +sudo kubeadm init phase upload-certs --upload-certs +``` + +5. Join `cp-2` and `cp-3`: + +```bash +sudo th-kubeadm-join-control-plane '' +``` + +6. Join workers: + +```bash +sudo th-kubeadm-join-worker '' +``` + +7. Validate from a control plane: + +```bash +kubectl get nodes -o wide +kubectl -n kube-system get pods -o wide +``` + ## Notes -- This does not run `kubeadm init/join` automatically. -- It prepares OS/runtime/kernel prerequisites so kubeadm bootstrapping is clean. +- Scripts are intentionally manual-triggered (predictable for homelab bring-up). +- If `.250` on the node subnet is already in use, change `controlPlaneVipSuffix` + in `modules/k8s-cluster-settings.nix` before bootstrap. diff --git a/nixos/kubeadm/hosts/cp-1.nix b/nixos/kubeadm/hosts/cp-1.nix index 56af9ba..3de50fc 100644 --- a/nixos/kubeadm/hosts/cp-1.nix +++ b/nixos/kubeadm/hosts/cp-1.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/cp-2.nix b/nixos/kubeadm/hosts/cp-2.nix index a186ca8..03b6fb7 100644 --- a/nixos/kubeadm/hosts/cp-2.nix +++ b/nixos/kubeadm/hosts/cp-2.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/cp-3.nix b/nixos/kubeadm/hosts/cp-3.nix index 5006fff..40acfd4 100644 --- a/nixos/kubeadm/hosts/cp-3.nix +++ b/nixos/kubeadm/hosts/cp-3.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-control-plane.nix ] diff --git a/nixos/kubeadm/hosts/wk-1.nix b/nixos/kubeadm/hosts/wk-1.nix index 709643e..d102e4c 100644 --- a/nixos/kubeadm/hosts/wk-1.nix +++ b/nixos/kubeadm/hosts/wk-1.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/hosts/wk-2.nix b/nixos/kubeadm/hosts/wk-2.nix index 5a11e90..78954db 100644 --- a/nixos/kubeadm/hosts/wk-2.nix +++ b/nixos/kubeadm/hosts/wk-2.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/hosts/wk-3.nix b/nixos/kubeadm/hosts/wk-3.nix index 64d11d5..1d21f36 100644 --- a/nixos/kubeadm/hosts/wk-3.nix +++ b/nixos/kubeadm/hosts/wk-3.nix @@ -3,6 +3,7 @@ { imports = [ + ../modules/k8s-cluster-settings.nix ../modules/k8s-common.nix ../modules/k8s-worker.nix ] diff --git a/nixos/kubeadm/modules/k8s-cluster-settings.nix b/nixos/kubeadm/modules/k8s-cluster-settings.nix new file mode 100644 index 0000000..2c89030 --- /dev/null +++ b/nixos/kubeadm/modules/k8s-cluster-settings.nix @@ -0,0 +1,12 @@ +{ ... }: + +{ + terrahome.kubeadm = { + k8sMinor = "1.31"; + controlPlaneInterface = "eth0"; + controlPlaneVipSuffix = 250; + podSubnet = "10.244.0.0/16"; + serviceSubnet = "10.96.0.0/12"; + clusterDomain = "cluster.local"; + }; +} diff --git a/nixos/kubeadm/modules/k8s-common.nix b/nixos/kubeadm/modules/k8s-common.nix index 2533868..a4f6c41 100644 --- a/nixos/kubeadm/modules/k8s-common.nix +++ b/nixos/kubeadm/modules/k8s-common.nix @@ -1,6 +1,43 @@ -{ pkgs, ... }: +{ config, lib, pkgs, ... }: +let + pinnedK8s = lib.attrByPath [ "kubernetes_1_31" ] pkgs.kubernetes pkgs; + kubeVipImage = "ghcr.io/kube-vip/kube-vip:v0.8.9"; +in { + options.terrahome.kubeadm = { + k8sMinor = lib.mkOption { + type = lib.types.str; + default = "1.31"; + }; + + controlPlaneInterface = lib.mkOption { + type = lib.types.str; + default = "eth0"; + }; + + controlPlaneVipSuffix = lib.mkOption { + type = lib.types.int; + default = 250; + }; + + podSubnet = lib.mkOption { + type = lib.types.str; + default = "10.244.0.0/16"; + }; + + serviceSubnet = lib.mkOption { + type = lib.types.str; + default = "10.96.0.0/12"; + }; + + clusterDomain = lib.mkOption { + type = lib.types.str; + default = "cluster.local"; + }; + }; + + config = { boot.kernelModules = [ "overlay" "br_netfilter" ]; boot.kernel.sysctl = { @@ -10,6 +47,11 @@ }; virtualisation.containerd.enable = true; + virtualisation.containerd.settings = { + plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = true; + }; + + swapDevices = lib.mkForce [ ]; services.openssh.enable = true; services.openssh.settings = { @@ -17,19 +59,143 @@ KbdInteractiveAuthentication = false; }; - environment.systemPackages = with pkgs; [ + environment.variables = { + KUBECONFIG = "/etc/kubernetes/admin.conf"; + KUBE_VIP_IMAGE = kubeVipImage; + }; + + environment.systemPackages = (with pkgs; [ containerd cri-tools cni-plugins - kubernetes - kubectl + pinnedK8s kubernetes-helm conntrack-tools socat ethtool ipvsadm + iproute2 + iptables + ebtables jq curl vim + gawk + ]) ++ [ + (pkgs.writeShellScriptBin "th-kubeadm-init" '' + set -euo pipefail + + iface="${config.terrahome.kubeadm.controlPlaneInterface}" + suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}" + pod_subnet="${config.terrahome.kubeadm.podSubnet}" + service_subnet="${config.terrahome.kubeadm.serviceSubnet}" + domain="${config.terrahome.kubeadm.clusterDomain}" + + local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') + if [ -z "${local_ip_cidr:-}" ]; then + echo "Could not determine IPv4 CIDR on interface $iface" + exit 1 + fi + + subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}') + vip="$subnet_prefix.$suffix" + + echo "Using control-plane endpoint: $vip:6443" + echo "Using kube-vip interface: $iface" + + mkdir -p /etc/kubernetes/manifests + ctr image pull "$KUBE_VIP_IMAGE" + + ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \ + --interface "$iface" \ + --address "$vip" \ + --controlplane \ + --services \ + --arp \ + --leaderElection \ + > /etc/kubernetes/manifests/kube-vip.yaml + + kubeadm init \ + --control-plane-endpoint "$vip:6443" \ + --upload-certs \ + --pod-network-cidr "$pod_subnet" \ + --service-cidr "$service_subnet" \ + --service-dns-domain "$domain" + + mkdir -p /root/.kube + cp /etc/kubernetes/admin.conf /root/.kube/config + chmod 600 /root/.kube/config + + echo + echo "Next: install Cilium, then generate join commands:" + echo " kubeadm token create --print-join-command" + echo " kubeadm token create --print-join-command --certificate-key " + '') + + (pkgs.writeShellScriptBin "th-kubeadm-join-control-plane" '' + set -euo pipefail + if [ "$#" -lt 1 ]; then + echo "Usage: th-kubeadm-join-control-plane ''" + exit 1 + fi + + iface="${config.terrahome.kubeadm.controlPlaneInterface}" + suffix="${toString config.terrahome.kubeadm.controlPlaneVipSuffix}" + local_ip_cidr=$(ip -4 -o addr show dev "$iface" | awk 'NR==1 {print $4}') + if [ -z "${local_ip_cidr:-}" ]; then + echo "Could not determine IPv4 CIDR on interface $iface" + exit 1 + fi + + subnet_prefix=$(echo "$local_ip_cidr" | cut -d/ -f1 | awk -F. '{print $1"."$2"."$3}') + vip="$subnet_prefix.$suffix" + + mkdir -p /etc/kubernetes/manifests + ctr image pull "$KUBE_VIP_IMAGE" + ctr run --rm --net-host "$KUBE_VIP_IMAGE" kube-vip /kube-vip manifest pod \ + --interface "$iface" \ + --address "$vip" \ + --controlplane \ + --services \ + --arp \ + --leaderElection \ + > /etc/kubernetes/manifests/kube-vip.yaml + + eval "$1" + '') + + (pkgs.writeShellScriptBin "th-kubeadm-join-worker" '' + set -euo pipefail + if [ "$#" -lt 1 ]; then + echo "Usage: th-kubeadm-join-worker ''" + exit 1 + fi + + eval "$1" + '') + + (pkgs.writeShellScriptBin "th-kubeadm-status" '' + set -euo pipefail + systemctl is-active containerd || true + systemctl is-active kubelet || true + crictl info >/dev/null && echo "crictl: ok" || echo "crictl: not-ready" + '') ]; + + systemd.services.kubelet = { + description = "Kubernetes Kubelet"; + wantedBy = [ "multi-user.target" ]; + after = [ "containerd.service" "network-online.target" ]; + serviceConfig = { + ExecStart = "${pinnedK8s}/bin/kubelet"; + Restart = "always"; + RestartSec = "10"; + }; + }; + + systemd.tmpfiles.rules = [ + "d /etc/kubernetes 0755 root root -" + "d /etc/kubernetes/manifests 0755 root root -" + ]; + }; }