feat: automate private tailnet access on cp1

README.md

@@ -229,11 +229,13 @@ Grafana and Prometheus are exposed through a single Tailscale front door backed
 
 ### Access Grafana and Prometheus
 
-Preferred (when Tailscale Operator is healthy):
+Preferred private access:
 
-- Grafana: `http://observability/grafana/` (or `http://observability.<your-tailnet>/grafana/`)
+- Grafana: `http://k8s-cluster-cp-1.<your-tailnet>:30080/`
-- Prometheus: `http://observability/prometheus/` (or `http://observability.<your-tailnet>/prometheus/`)
+- Prometheus: `http://k8s-cluster-cp-1.<your-tailnet>:30990/`
-- Flux UI: `http://observability:9001/` (or `http://observability.<your-tailnet>:9001/`)
+- Flux UI: `http://k8s-cluster-cp-1.<your-tailnet>:30901/`
+
+This access path is bootstrapped automatically by Ansible on `control_plane[0]` using persistent `kubectl port-forward` systemd services plus `tailscale serve`, so it survives cluster rebuilds.
 
 Fallback (port-forward from a tailnet-connected machine):
 

ansible/roles/private-access/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Create systemd unit for Grafana private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-grafana.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Grafana for Tailscale access
+    unit_namespace: observability
+    unit_target: svc/observability-kube-prometheus-stack-grafana
+    unit_local_port: 13080
+    unit_remote_port: 80
+
+- name: Create systemd unit for Prometheus private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-prometheus.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Prometheus for Tailscale access
+    unit_namespace: observability
+    unit_target: svc/observability-kube-prometh-prometheus
+    unit_local_port: 19090
+    unit_remote_port: 9090
+
+- name: Create systemd unit for Flux UI private access
+  template:
+    src: kubectl-port-forward.service.j2
+    dest: /etc/systemd/system/k8s-portforward-flux-ui.service
+    mode: "0644"
+  vars:
+    unit_description: Port-forward Flux UI for Tailscale access
+    unit_namespace: flux-system
+    unit_target: svc/flux-system-weave-gitops
+    unit_local_port: 19001
+    unit_remote_port: 9001
+
+- name: Reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: Enable and start private access port-forward services
+  systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - k8s-portforward-grafana.service
+    - k8s-portforward-prometheus.service
+    - k8s-portforward-flux-ui.service
+
+- name: Configure Tailscale Serve for private access endpoints
+  shell: >-
+    tailscale serve reset &&
+    tailscale serve --bg --tcp={{ private_access_grafana_port }} tcp://127.0.0.1:13080 &&
+    tailscale serve --bg --tcp={{ private_access_prometheus_port }} tcp://127.0.0.1:19090 &&
+    tailscale serve --bg --tcp={{ private_access_flux_port }} tcp://127.0.0.1:19001
+  changed_when: true

ansible/roles/private-access/templates/kubectl-port-forward.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description={{ unit_description }}
+After=network-online.target k3s.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=5
+ExecStart=/usr/local/bin/kubectl -n {{ unit_namespace }} port-forward --address 127.0.0.1 {{ unit_target }} {{ unit_local_port }}:{{ unit_remote_port }}
+
+[Install]
+WantedBy=multi-user.target

ansible/site.yml

@@ -112,6 +112,17 @@
     - role: observability-content
       when: not (observability_gitops_enabled | default(true) | bool)
 
+- name: Configure private tailnet access
+  hosts: control_plane[0]
+  become: true
+  vars:
+    private_access_grafana_port: 30080
+    private_access_prometheus_port: 30990
+    private_access_flux_port: 30901
+
+  roles:
+    - private-access
+
 - name: Finalize
   hosts: localhost
   connection: local