From c9be2a2fc8661753130fd8880791cbeb94e84a2c Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 01:10:19 +0000 Subject: [PATCH 1/4] fix: align VM boot disk and add Terraform safety workflows Switch VM boot order/disks to scsi0 to match cloned NixOS template boot layout, add destroy guards to plan/apply workflows, and replace destroy workflow with a confirmed manual dispatch nuke flow that uses remote B2 state. --- .gitea/workflows/terraform-apply.yml | 17 +++++- .gitea/workflows/terraform-destroy.yml | 83 +++++++++++++++++++++----- .gitea/workflows/terraform-plan.yml | 13 ++++ terraform/main.tf | 28 +++++---- 4 files changed, 111 insertions(+), 30 deletions(-) diff --git a/.gitea/workflows/terraform-apply.yml b/.gitea/workflows/terraform-apply.yml index a94816c..4e6e49a 100644 --- a/.gitea/workflows/terraform-apply.yml +++ b/.gitea/workflows/terraform-apply.yml @@ -47,11 +47,24 @@ jobs: - name: Terraform Plan working-directory: terraform - run: terraform plan + run: terraform plan -out=tfplan + + - name: Block accidental destroy + env: + ALLOW_TF_DESTROY: ${{ secrets.ALLOW_TF_DESTROY }} + working-directory: terraform + run: | + terraform show -json tfplan > tfplan.json + DESTROY_COUNT=$(python3 -c 'import json; p=json.load(open("tfplan.json")); print(sum(1 for rc in p.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + echo "Planned deletes: $DESTROY_COUNT" + if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then + echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." + exit 1 + fi - name: Terraform Apply working-directory: terraform - run: terraform apply -auto-approve + run: terraform apply -auto-approve tfplan - name: Enroll VMs in Tailscale env: diff --git a/.gitea/workflows/terraform-destroy.yml b/.gitea/workflows/terraform-destroy.yml index f082b60..ad4245a 100644 --- a/.gitea/workflows/terraform-destroy.yml +++ b/.gitea/workflows/terraform-destroy.yml @@ -1,28 +1,61 @@ -name: Gitea Destroy Terraform -run-name: ${{ gitea.actor }} triggered a Terraform Destroy 🧨 +name: Terraform Destroy +run-name: ${{ gitea.actor }} requested Terraform destroy on: - workflow_dispatch: # Manual trigger + workflow_dispatch: + inputs: + confirm: + description: "Type NUKE to confirm destroy" + required: true + type: string + target: + description: "Destroy scope" + required: true + default: all + type: choice + options: + - all + - alpacas + - llamas jobs: destroy: name: "Terraform Destroy" runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - - env: - TF_VAR_SSH_KEY: ${{ secrets.TF_VAR_SSH_KEY_PUBLIC }} - TF_VAR_TS_AUTHKEY: ${{ secrets.TF_VAR_TS_AUTHKEY }} - TF_VAR_PROXMOX_PASSWORD: ${{ secrets.TF_VAR_PROXMOX_PASSWORD }} - - steps: + - name: Validate confirmation phrase + run: | + if [ "${{ inputs.confirm }}" != "NUKE" ]; then + echo "Confirmation failed. You must type NUKE." + exit 1 + fi + - name: Checkout repository uses: actions/checkout@v4 + - name: Create Terraform secret files + working-directory: terraform + run: | + cat > secrets.auto.tfvars << EOF + pm_api_token_secret = "${{ secrets.PM_API_TOKEN_SECRET }}" + EOF + cat > backend.hcl << EOF + bucket = "${{ secrets.B2_TF_BUCKET }}" + key = "terraform.tfstate" + region = "us-east-005" + endpoints = { + s3 = "${{ secrets.B2_TF_ENDPOINT }}" + } + access_key = "$(printf '%s' "${{ secrets.B2_KEY_ID }}" | tr -d '\r\n')" + secret_key = "$(printf '%s' "${{ secrets.B2_APPLICATION_KEY }}" | tr -d '\r\n')" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_path_style = true + EOF + - name: Set up Terraform uses: hashicorp/setup-terraform@v2 with: @@ -30,9 +63,27 @@ jobs: - name: Terraform Init working-directory: terraform - run: terraform init + run: terraform init -reconfigure -backend-config=backend.hcl - - name: Terraform Destroy + - name: Terraform Destroy Plan working-directory: terraform - run: terraform destroy -auto-approve + run: | + case "${{ inputs.target }}" in + all) + terraform plan -destroy -out=tfdestroy + ;; + alpacas) + terraform plan -destroy -target=proxmox_vm_qemu.alpacas -out=tfdestroy + ;; + llamas) + terraform plan -destroy -target=proxmox_vm_qemu.llamas -out=tfdestroy + ;; + *) + echo "Invalid destroy target: ${{ inputs.target }}" + exit 1 + ;; + esac + - name: Terraform Destroy Apply + working-directory: terraform + run: terraform apply -auto-approve tfdestroy diff --git a/.gitea/workflows/terraform-plan.yml b/.gitea/workflows/terraform-plan.yml index 7854dd6..b54fa26 100644 --- a/.gitea/workflows/terraform-plan.yml +++ b/.gitea/workflows/terraform-plan.yml @@ -63,6 +63,19 @@ jobs: working-directory: terraform run: terraform plan -out=tfplan + - name: Block accidental destroy + env: + ALLOW_TF_DESTROY: ${{ secrets.ALLOW_TF_DESTROY }} + working-directory: terraform + run: | + terraform show -json tfplan > tfplan.json + DESTROY_COUNT=$(python3 -c 'import json; p=json.load(open("tfplan.json")); print(sum(1 for rc in p.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + echo "Planned deletes: $DESTROY_COUNT" + if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then + echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." + exit 1 + fi + - name: Upload Terraform Plan uses: actions/upload-artifact@v3 with: diff --git a/terraform/main.tf b/terraform/main.tf index 8631a17..2c5b727 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -26,19 +26,21 @@ resource "proxmox_vm_qemu" "alpacas" { os_type = "cloud-init" agent = 1 - sockets = var.sockets - cores = var.cores + cpu { + sockets = var.sockets + cores = var.cores + } memory = var.memory scsihw = "virtio-scsi-pci" - boot = "order=virtio0" - bootdisk = "virtio0" + boot = "order=scsi0" + bootdisk = "scsi0" ipconfig0 = "ip=dhcp" cicustom = "user=local:snippets/cloud_init_global.yaml" disks { - virtio { - virtio0 { + scsi { + scsi0 { disk { size = var.disk_size storage = var.storage @@ -73,18 +75,20 @@ resource "proxmox_vm_qemu" "llamas" { os_type = "cloud-init" agent = 1 - sockets = var.sockets - cores = var.cores + cpu { + sockets = var.sockets + cores = var.cores + } memory = var.memory scsihw = "virtio-scsi-pci" - boot = "order=virtio0" - bootdisk = "virtio0" + boot = "order=scsi0" + bootdisk = "scsi0" ipconfig0 = "ip=dhcp" cicustom = "user=local:snippets/cloud_init_global.yaml" disks { - virtio { - virtio0 { + scsi { + scsi0 { disk { size = var.disk_size storage = var.storage -- 2.49.1 From afe19041d91926ad39afd1a13f423fd62d1d67d9 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 01:16:19 +0000 Subject: [PATCH 2/4] fix: make destroy guard parse tfplan JSON robustly Use terraform show with no-color and resilient JSON extraction to avoid parser failures when workflow output includes non-JSON noise. --- .gitea/workflows/terraform-apply.yml | 4 ++-- .gitea/workflows/terraform-plan.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/terraform-apply.yml b/.gitea/workflows/terraform-apply.yml index 4e6e49a..d43e4d5 100644 --- a/.gitea/workflows/terraform-apply.yml +++ b/.gitea/workflows/terraform-apply.yml @@ -54,8 +54,8 @@ jobs: ALLOW_TF_DESTROY: ${{ secrets.ALLOW_TF_DESTROY }} working-directory: terraform run: | - terraform show -json tfplan > tfplan.json - DESTROY_COUNT=$(python3 -c 'import json; p=json.load(open("tfplan.json")); print(sum(1 for rc in p.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + terraform show -json -no-color tfplan > tfplan.json + DESTROY_COUNT=$(python3 -c 'import json,sys; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); end=raw.rfind("}"); data=json.loads(raw[start:end+1]); print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') echo "Planned deletes: $DESTROY_COUNT" if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." diff --git a/.gitea/workflows/terraform-plan.yml b/.gitea/workflows/terraform-plan.yml index b54fa26..84d12ba 100644 --- a/.gitea/workflows/terraform-plan.yml +++ b/.gitea/workflows/terraform-plan.yml @@ -68,8 +68,8 @@ jobs: ALLOW_TF_DESTROY: ${{ secrets.ALLOW_TF_DESTROY }} working-directory: terraform run: | - terraform show -json tfplan > tfplan.json - DESTROY_COUNT=$(python3 -c 'import json; p=json.load(open("tfplan.json")); print(sum(1 for rc in p.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + terraform show -json -no-color tfplan > tfplan.json + DESTROY_COUNT=$(python3 -c 'import json,sys; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); end=raw.rfind("}"); data=json.loads(raw[start:end+1]); print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') echo "Planned deletes: $DESTROY_COUNT" if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." -- 2.49.1 From d1a7ccc98cfeb6266c78f5518e5dc499a396559b Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 01:17:51 +0000 Subject: [PATCH 3/4] chore: serialize Terraform workflows to prevent races Add global workflow concurrency group with queueing enabled so plan/apply/destroy runs do not overlap and contend for shared remote state. --- .gitea/workflows/terraform-apply.yml | 4 ++++ .gitea/workflows/terraform-destroy.yml | 4 ++++ .gitea/workflows/terraform-plan.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.gitea/workflows/terraform-apply.yml b/.gitea/workflows/terraform-apply.yml index d43e4d5..49819af 100644 --- a/.gitea/workflows/terraform-apply.yml +++ b/.gitea/workflows/terraform-apply.yml @@ -5,6 +5,10 @@ on: branches: - master +concurrency: + group: terraform-global + cancel-in-progress: false + jobs: terraform: name: "Terraform Apply" diff --git a/.gitea/workflows/terraform-destroy.yml b/.gitea/workflows/terraform-destroy.yml index ad4245a..9326660 100644 --- a/.gitea/workflows/terraform-destroy.yml +++ b/.gitea/workflows/terraform-destroy.yml @@ -18,6 +18,10 @@ on: - alpacas - llamas +concurrency: + group: terraform-global + cancel-in-progress: false + jobs: destroy: name: "Terraform Destroy" diff --git a/.gitea/workflows/terraform-plan.yml b/.gitea/workflows/terraform-plan.yml index 84d12ba..b5c8f8f 100644 --- a/.gitea/workflows/terraform-plan.yml +++ b/.gitea/workflows/terraform-plan.yml @@ -6,6 +6,10 @@ on: - stage - test +concurrency: + group: terraform-global + cancel-in-progress: false + jobs: terraform: name: "Terraform Plan" -- 2.49.1 From a7f68c0c4b67467eec72b2713992bed585e72626 Mon Sep 17 00:00:00 2001 From: MichaelFisher1997 Date: Sat, 28 Feb 2026 01:23:07 +0000 Subject: [PATCH 4/4] fix: tolerate extra output in destroy guard parser Parse the first JSON object from terraform show output to avoid failures when extra non-JSON lines are present. --- .gitea/workflows/terraform-apply.yml | 2 +- .gitea/workflows/terraform-plan.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/terraform-apply.yml b/.gitea/workflows/terraform-apply.yml index 49819af..7e8ec86 100644 --- a/.gitea/workflows/terraform-apply.yml +++ b/.gitea/workflows/terraform-apply.yml @@ -59,7 +59,7 @@ jobs: working-directory: terraform run: | terraform show -json -no-color tfplan > tfplan.json - DESTROY_COUNT=$(python3 -c 'import json,sys; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); end=raw.rfind("}"); data=json.loads(raw[start:end+1]); print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + DESTROY_COUNT=$(python3 -c 'import json; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); data=json.JSONDecoder().raw_decode(raw[start:])[0]; print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') echo "Planned deletes: $DESTROY_COUNT" if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." diff --git a/.gitea/workflows/terraform-plan.yml b/.gitea/workflows/terraform-plan.yml index b5c8f8f..a213eda 100644 --- a/.gitea/workflows/terraform-plan.yml +++ b/.gitea/workflows/terraform-plan.yml @@ -73,7 +73,7 @@ jobs: working-directory: terraform run: | terraform show -json -no-color tfplan > tfplan.json - DESTROY_COUNT=$(python3 -c 'import json,sys; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); end=raw.rfind("}"); data=json.loads(raw[start:end+1]); print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') + DESTROY_COUNT=$(python3 -c 'import json; raw=open("tfplan.json","rb").read().decode("utf-8","ignore"); start=raw.find("{"); data=json.JSONDecoder().raw_decode(raw[start:])[0]; print(sum(1 for rc in data.get("resource_changes", []) if "delete" in rc.get("change", {}).get("actions", [])))') echo "Planned deletes: $DESTROY_COUNT" if [ "$DESTROY_COUNT" -gt 0 ] && [ "${ALLOW_TF_DESTROY}" != "true" ]; then echo "Destroy actions detected. Set ALLOW_TF_DESTROY=true to allow." -- 2.49.1