From f95e0051a55a8f165b1169f66ab48594c3be32ec Mon Sep 17 00:00:00 2001
From: MichaelFisher1997 <contact@michaelfisher.tech>
Date: Sun, 8 Mar 2026 04:16:06 +0000
Subject: [PATCH] feat: automate private tailnet access on cp1

---
 README.md                                     | 10 ++--
 ansible/roles/private-access/tasks/main.yml   | 58 +++++++++++++++++++
 .../templates/kubectl-port-forward.service.j2 | 13 +++++
 ansible/site.yml                              | 11 ++++
 4 files changed, 88 insertions(+), 4 deletions(-)
 create mode 100644 ansible/roles/private-access/tasks/main.yml
 create mode 100644 ansible/roles/private-access/templates/kubectl-port-forward.service.j2

diff --git a/README.md b/README.md
index 4edaa05..50dc5bd 100644
--- a/README.md
+++ b/README.md
@@ -229,11 +229,13 @@ Grafana and Prometheus are exposed through a single Tailscale front door backed
 
 ### Access Grafana and Prometheus
 
-Preferred (when Tailscale Operator is healthy):
+Preferred private access:
 
-- Grafana: `http://observability/grafana/` (or `http://observability.<your-tailnet>/grafana/`)
-- Prometheus: `http://observability/prometheus/` (or `http://observability.<your-tailnet>/prometheus/`)
-- Flux UI: `http://observability:9001/` (or `http://observability.<your-tailnet>:9001/`)
+- Grafana: `http://k8s-cluster-cp-1.<your-tailnet>:30080/`
+- Prometheus: `http://k8s-cluster-cp-1.<your-tailnet>:30990/`
+- Flux UI: `http://k8s-cluster-cp-1.<your-tailnet>:30901/`
+
+This access path is bootstrapped automatically by Ansible on `control_plane[0]` using persistent `kubectl port-forward` systemd services plus `tailscale serve`, so it survives cluster rebuilds.
 
 Fallback (port-forward from a tailnet-connected machine):
 
diff --git a/ansible/roles/private-access/tasks/main.yml b/ansible/roles/private-access/tasks/main.yml
new file mode 100644
index 0000000..2160eb0
--- /dev/null
+++ b/ansible/roles/private-access/tasks/main.yml
@@ -0,0 +1,58 @@
+---
+- name: Create systemd unit for Grafana private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-grafana.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Grafana for Tailscale access
+    unit_namespace: observability
+    unit_target: svc/observability-kube-prometheus-stack-grafana
+    unit_local_port: 13080
+    unit_remote_port: 80
+
+- name: Create systemd unit for Prometheus private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-prometheus.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Prometheus for Tailscale access
+    unit_namespace: observability
+    unit_target: svc/observability-kube-prometh-prometheus
+    unit_local_port: 19090
+    unit_remote_port: 9090
+
+- name: Create systemd unit for Flux UI private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-flux-ui.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Flux UI for Tailscale access
+    unit_namespace: flux-system
+    unit_target: svc/flux-system-weave-gitops
+    unit_local_port: 19001
+    unit_remote_port: 9001
+
+- name: Reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: Enable and start private access port-forward services
+  systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - k8s-portforward-grafana.service
+    - k8s-portforward-prometheus.service
+    - k8s-portforward-flux-ui.service
+
+- name: Configure Tailscale Serve for private access endpoints
+  shell: >-
+    tailscale serve reset &&
+    tailscale serve --bg --tcp={{ private_access_grafana_port }} tcp://127.0.0.1:13080 &&
+    tailscale serve --bg --tcp={{ private_access_prometheus_port }} tcp://127.0.0.1:19090 &&
+    tailscale serve --bg --tcp={{ private_access_flux_port }} tcp://127.0.0.1:19001
+  changed_when: true
diff --git a/ansible/roles/private-access/templates/kubectl-port-forward.service.j2 b/ansible/roles/private-access/templates/kubectl-port-forward.service.j2
new file mode 100644
index 0000000..6abb573
--- /dev/null
+++ b/ansible/roles/private-access/templates/kubectl-port-forward.service.j2
@@ -0,0 +1,13 @@
+[Unit]
+Description={{ unit_description }}
+After=network-online.target k3s.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=5
+ExecStart=/usr/local/bin/kubectl -n {{ unit_namespace }} port-forward --address 127.0.0.1 {{ unit_target }} {{ unit_local_port }}:{{ unit_remote_port }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/site.yml b/ansible/site.yml
index 0e7ee6e..3cffcf5 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -112,6 +112,17 @@
     - role: observability-content
       when: not (observability_gitops_enabled | default(true) | bool)
 
+- name: Configure private tailnet access
+  hosts: control_plane[0]
+  become: true
+  vars:
+    private_access_grafana_port: 30080
+    private_access_prometheus_port: 30990
+    private_access_flux_port: 30901
+
+  roles:
+    - private-access
+
 - name: Finalize
   hosts: localhost
   connection: local