fix: ignore stale SSH host keys for ephemeral homelab VMs

Fresh destroy/recreate cycles change VM host keys, which was breaking bootstrap after rebuilds. Use a disposable known-hosts policy in the controller SSH options so automation does not fail on expected key rotation.

.gitea/workflows/kubeadm-bootstrap.yml

@@ -27,7 +27,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create SSH key
         run: |

.gitea/workflows/kubeadm-reset.yml

@@ -27,7 +27,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create SSH key
         run: |

.gitea/workflows/terraform-apply.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create secrets.tfvars
         working-directory: terraform

.gitea/workflows/terraform-destroy.yml

@@ -36,7 +36,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create Terraform secret files
         working-directory: terraform
@@ -77,13 +77,13 @@ jobs:
           set -euo pipefail
           case "${{ inputs.target }}" in
             all)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -out=tfdestroy"
               ;;
             control-planes)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -target=proxmox_vm_qemu.control_planes -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -target=proxmox_vm_qemu.control_planes -out=tfdestroy"
               ;;
             workers)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -target=proxmox_vm_qemu.workers -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -target=proxmox_vm_qemu.workers -out=tfdestroy"
               ;;
             *)
               echo "Invalid destroy target: ${{ inputs.target }}"

.gitea/workflows/terraform-plan.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create secrets.tfvars
         working-directory: terraform

nixos/kubeadm/bootstrap/controller.py

@@ -110,7 +110,9 @@ class Controller:
             "-o",
             "IdentitiesOnly=yes",
             "-o",
-            "StrictHostKeyChecking=accept-new",
+            "StrictHostKeyChecking=no",
+            "-o",
+            "UserKnownHostsFile=/dev/null",
             "-i",
             self.ssh_key,
         ]
@@ -121,6 +123,8 @@ class Controller:
         self.fast_mode = self.env.get("FAST_MODE", "1")
         self.skip_rebuild = self.env.get("SKIP_REBUILD", "0") == "1"
         self.force_reinit = True
+        self.ssh_ready_retries = int(self.env.get("SSH_READY_RETRIES", "20"))
+        self.ssh_ready_delay = int(self.env.get("SSH_READY_DELAY_SEC", "15"))
 
     def log(self, msg):
         print(f"==> {msg}")
@@ -130,13 +134,26 @@ class Controller:
         return run_local(full, check=check, capture=True)
 
     def detect_user(self, ip):
-        for user in self.ssh_candidates:
-            proc = self._ssh(user, ip, "true", check=False)
-            if proc.returncode == 0:
-                self.active_ssh_user = user
-                self.log(f"Using SSH user '{user}' for {ip}")
-                return
-        raise RuntimeError(f"Unable to authenticate to {ip} with users: {', '.join(self.ssh_candidates)}")
+        for attempt in range(1, self.ssh_ready_retries + 1):
+            for user in self.ssh_candidates:
+                proc = self._ssh(user, ip, "true", check=False)
+                if proc.returncode == 0:
+                    self.active_ssh_user = user
+                    self.log(f"Using SSH user '{user}' for {ip}")
+                    return
+            if attempt < self.ssh_ready_retries:
+                self.log(
+                    f"SSH not ready on {ip} yet; retrying in {self.ssh_ready_delay}s "
+                    f"({attempt}/{self.ssh_ready_retries})"
+                )
+                time.sleep(self.ssh_ready_delay)
+        raise RuntimeError(
+            "Unable to authenticate to "
+            f"{ip} with users: {', '.join(self.ssh_candidates)}. "
+            "If this is a freshly cloned VM, the Proxmox source template likely does not yet include the "
+            "current cloud-init-capable NixOS template configuration from nixos/template-base. "
+            "Terraform can only clone what exists in Proxmox; it cannot retrofit cloud-init support into an old template."
+        )
 
     def remote(self, ip, cmd, check=True):
         ordered = [self.active_ssh_user] + [u for u in self.ssh_candidates if u != self.active_ssh_user]
@@ -157,14 +174,7 @@ class Controller:
         return last
 
     def prepare_known_hosts(self):
-        ssh_dir = Path.home() / ".ssh"
-        ssh_dir.mkdir(parents=True, exist_ok=True)
-        (ssh_dir / "known_hosts").touch()
-        run_local(["chmod", "700", str(ssh_dir)])
-        run_local(["chmod", "600", str(ssh_dir / "known_hosts")])
-        for ip in self.node_ips.values():
-            run_local(["ssh-keygen", "-R", ip], check=False)
-            run_local(f"ssh-keyscan -H {shlex.quote(ip)} >> {shlex.quote(str(ssh_dir / 'known_hosts'))}", check=False)
+        pass
 
     def prepare_remote_nix(self, ip):
         self.remote(ip, "sudo mkdir -p /etc/nix")
@@ -265,11 +275,42 @@ class Controller:
 
     def stage_install_cni(self):
         self.log("Installing Flannel")
+        manifest_path = self.script_dir.parent / "manifests" / "kube-flannel.yml"
+        manifest_b64 = base64.b64encode(manifest_path.read_bytes()).decode()
+
         self.remote(
             self.primary_ip,
-            "sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/flannel-io/flannel/v0.25.5/Documentation/kube-flannel.yml",
+            (
+                "sudo mkdir -p /var/lib/terrahome && "
+                f"echo {shlex.quote(manifest_b64)} | base64 -d | sudo tee /var/lib/terrahome/kube-flannel.yml >/dev/null"
+            ),
         )
 
+        self.log("Waiting for API readiness before applying Flannel")
+        ready = False
+        for _ in range(30):
+            if self.cluster_ready():
+                ready = True
+                break
+            time.sleep(10)
+        if not ready:
+            raise RuntimeError("API server did not become ready before Flannel install")
+
+        last_error = None
+        for attempt in range(1, 6):
+            proc = self.remote(
+                self.primary_ip,
+                "sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /var/lib/terrahome/kube-flannel.yml",
+                check=False,
+            )
+            if proc.returncode == 0:
+                return
+            last_error = (proc.stdout or "") + ("\n" if proc.stdout and proc.stderr else "") + (proc.stderr or "")
+            self.log(f"Flannel apply attempt {attempt}/5 failed; retrying in 15s")
+            time.sleep(15)
+
+        raise RuntimeError(f"Flannel apply failed after retries\n{last_error or ''}")
+
     def cluster_has_node(self, name):
         cmd = f"sudo kubectl --kubeconfig /etc/kubernetes/admin.conf get node {shlex.quote(name)} >/dev/null 2>&1"
         return self.remote(self.primary_ip, cmd, check=False).returncode == 0
@@ -331,6 +372,18 @@ class Controller:
                 check=False,
             )
             print(proc.stdout)
+            proc = self.remote(
+                self.primary_ip,
+                "for p in $(sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel get pods -o name 2>/dev/null); do echo \"--- describe $p ---\"; sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel describe $p || true; done",
+                check=False,
+            )
+            print(proc.stdout)
+            proc = self.remote(
+                self.primary_ip,
+                "for p in $(sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel get pods -o name 2>/dev/null); do echo \"--- logs $p kube-flannel ---\"; sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel logs $p -c kube-flannel --tail=120 || true; echo \"--- logs $p install-cni-plugin ---\"; sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel logs $p -c install-cni-plugin --tail=120 || true; echo \"--- logs $p install-cni ---\"; sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel logs $p -c install-cni --tail=120 || true; done",
+                check=False,
+            )
+            print(proc.stdout)
             proc = self.remote(
                 self.primary_ip,
                 "for p in $(sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel get pods -o name 2>/dev/null); do sudo kubectl --kubeconfig /etc/kubernetes/admin.conf -n kube-flannel logs --tail=120 $p || true; done",

nixos/kubeadm/manifests/kube-flannel.yml

@@ -0,0 +1,212 @@
+---
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: kube-flannel
+  labels:
+    k8s-app: flannel
+    pod-security.kubernetes.io/enforce: privileged
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: flannel
+  name: flannel
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: flannel
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-flannel
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: flannel
+  name: flannel
+  namespace: kube-flannel
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-flannel
+  labels:
+    tier: node
+    k8s-app: flannel
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "10.244.0.0/16",
+      "EnableNFTables": false,
+      "Backend": {
+        "Type": "vxlan"
+      }
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: kube-flannel
+  labels:
+    tier: node
+    app: flannel
+    k8s-app: flannel
+spec:
+  selector:
+    matchLabels:
+      app: flannel
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      initContainers:
+      - name: install-cni-plugin
+        image: docker.io/flannel/flannel-cni-plugin:v1.5.1-flannel1
+        command:
+        - cp
+        args:
+        - -f
+        - /flannel
+        - /opt/cni/bin/flannel
+        volumeMounts:
+        - name: cni-plugin
+          mountPath: /opt/cni/bin
+      - name: install-cni
+        image: docker.io/flannel/flannel:v0.25.5
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      containers:
+      - name: kube-flannel
+        image: docker.io/flannel/flannel:v0.25.5
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq
+        - --kube-subnet-mgr
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: false
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: EVENT_QUEUE_DEPTH
+          value: "5000"
+        volumeMounts:
+        - name: run
+          mountPath: /run/flannel
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+      volumes:
+      - name: run
+        hostPath:
+          path: /run/flannel
+          type: DirectoryOrCreate
+      - name: cni-plugin
+        hostPath:
+          path: /opt/cni/bin
+          type: DirectoryOrCreate
+      - name: cni
+        hostPath:
+          path: /etc/cni/net.d
+          type: DirectoryOrCreate
+      - name: flannel-cfg
+        configMap:
+          name: kube-flannel-cfg
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate

nixos/kubeadm/modules/k8s-common.nix

@@ -384,6 +384,7 @@ in
   systemd.services.kubelet = {
     description = "Kubernetes Kubelet";
     wantedBy = [ "multi-user.target" ];
+    path = [ pkgs.util-linux ];
     wants = [ "network-online.target" ];
     after = [ "containerd.service" "network-online.target" ];
     serviceConfig = {
@@ -409,6 +410,9 @@ in
   systemd.tmpfiles.rules = [
     "d /etc/kubernetes 0755 root root -"
     "d /etc/kubernetes/manifests 0755 root root -"
+    "d /etc/cni/net.d 0755 root root -"
+    "d /opt/cni/bin 0755 root root -"
+    "d /run/flannel 0755 root root -"
     "d /var/lib/kubelet 0755 root root -"
     "d /var/lib/kubelet/pki 0755 root root -"
   ];

nixos/template-base/configuration.nix

@@ -11,6 +11,7 @@ in
 
   networking.hostName = "k8s-base-template";
   networking.useDHCP = lib.mkDefault true;
+  networking.useNetworkd = true;
   networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
 
   boot.loader.systemd-boot.enable = lib.mkForce false;
@@ -20,6 +21,8 @@ in
   };
 
   services.qemuGuest.enable = true;
+  services.cloud-init.enable = true;
+  services.cloud-init.network.enable = true;
   services.openssh.enable = true;
   services.openssh.settings = {
     PasswordAuthentication = false;