Merge pull request 'fix: ignore stale SSH host keys for ephemeral homelab VMs' (#130) from stage into master

Reviewed-on: #130

.gitea/workflows/kubeadm-bootstrap.yml

@@ -27,7 +27,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create SSH key
         run: |

.gitea/workflows/kubeadm-reset.yml

@@ -27,7 +27,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create SSH key
         run: |

.gitea/workflows/terraform-apply.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create secrets.tfvars
         working-directory: terraform

.gitea/workflows/terraform-destroy.yml

@@ -36,7 +36,7 @@ jobs:
           fi
 
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create Terraform secret files
         working-directory: terraform
@@ -77,13 +77,13 @@ jobs:
           set -euo pipefail
           case "${{ inputs.target }}" in
             all)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -out=tfdestroy"
               ;;
             control-planes)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -target=proxmox_vm_qemu.control_planes -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -target=proxmox_vm_qemu.control_planes -out=tfdestroy"
               ;;
             workers)
-              TF_PLAN_CMD="terraform plan -parallelism=1 -destroy -target=proxmox_vm_qemu.workers -out=tfdestroy"
+              TF_PLAN_CMD="terraform plan -refresh=false -parallelism=1 -destroy -target=proxmox_vm_qemu.workers -out=tfdestroy"
               ;;
             *)
               echo "Invalid destroy target: ${{ inputs.target }}"

.gitea/workflows/terraform-plan.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: https://gitea.com/actions/checkout@v4
+        uses: actions/checkout@v4
 
       - name: Create secrets.tfvars
         working-directory: terraform

nixos/kubeadm/bootstrap/controller.py

@@ -110,7 +110,9 @@ class Controller:
             "-o",
             "IdentitiesOnly=yes",
             "-o",
-            "StrictHostKeyChecking=accept-new",
+            "StrictHostKeyChecking=no",
+            "-o",
+            "UserKnownHostsFile=/dev/null",
             "-i",
             self.ssh_key,
         ]
@@ -121,6 +123,8 @@ class Controller:
         self.fast_mode = self.env.get("FAST_MODE", "1")
         self.skip_rebuild = self.env.get("SKIP_REBUILD", "0") == "1"
         self.force_reinit = True
+        self.ssh_ready_retries = int(self.env.get("SSH_READY_RETRIES", "20"))
+        self.ssh_ready_delay = int(self.env.get("SSH_READY_DELAY_SEC", "15"))
 
     def log(self, msg):
         print(f"==> {msg}")
@@ -130,13 +134,26 @@ class Controller:
         return run_local(full, check=check, capture=True)
 
     def detect_user(self, ip):
-        for user in self.ssh_candidates:
-            proc = self._ssh(user, ip, "true", check=False)
-            if proc.returncode == 0:
-                self.active_ssh_user = user
-                self.log(f"Using SSH user '{user}' for {ip}")
-                return
-        raise RuntimeError(f"Unable to authenticate to {ip} with users: {', '.join(self.ssh_candidates)}")
+        for attempt in range(1, self.ssh_ready_retries + 1):
+            for user in self.ssh_candidates:
+                proc = self._ssh(user, ip, "true", check=False)
+                if proc.returncode == 0:
+                    self.active_ssh_user = user
+                    self.log(f"Using SSH user '{user}' for {ip}")
+                    return
+            if attempt < self.ssh_ready_retries:
+                self.log(
+                    f"SSH not ready on {ip} yet; retrying in {self.ssh_ready_delay}s "
+                    f"({attempt}/{self.ssh_ready_retries})"
+                )
+                time.sleep(self.ssh_ready_delay)
+        raise RuntimeError(
+            "Unable to authenticate to "
+            f"{ip} with users: {', '.join(self.ssh_candidates)}. "
+            "If this is a freshly cloned VM, the Proxmox source template likely does not yet include the "
+            "current cloud-init-capable NixOS template configuration from nixos/template-base. "
+            "Terraform can only clone what exists in Proxmox; it cannot retrofit cloud-init support into an old template."
+        )
 
     def remote(self, ip, cmd, check=True):
         ordered = [self.active_ssh_user] + [u for u in self.ssh_candidates if u != self.active_ssh_user]
@@ -157,14 +174,7 @@ class Controller:
         return last
 
     def prepare_known_hosts(self):
-        ssh_dir = Path.home() / ".ssh"
-        ssh_dir.mkdir(parents=True, exist_ok=True)
-        (ssh_dir / "known_hosts").touch()
-        run_local(["chmod", "700", str(ssh_dir)])
-        run_local(["chmod", "600", str(ssh_dir / "known_hosts")])
-        for ip in self.node_ips.values():
-            run_local(["ssh-keygen", "-R", ip], check=False)
-            run_local(f"ssh-keyscan -H {shlex.quote(ip)} >> {shlex.quote(str(ssh_dir / 'known_hosts'))}", check=False)
+        pass
 
     def prepare_remote_nix(self, ip):
         self.remote(ip, "sudo mkdir -p /etc/nix")

nixos/template-base/configuration.nix

@@ -11,6 +11,7 @@ in
 
   networking.hostName = "k8s-base-template";
   networking.useDHCP = lib.mkDefault true;
+  networking.useNetworkd = true;
   networking.nameservers = [ "1.1.1.1" "8.8.8.8" ];
 
   boot.loader.systemd-boot.enable = lib.mkForce false;
@@ -20,6 +21,8 @@ in
   };
 
   services.qemuGuest.enable = true;
+  services.cloud-init.enable = true;
+  services.cloud-init.network.enable = true;
   services.openssh.enable = true;
   services.openssh.settings = {
     PasswordAuthentication = false;