feat: Expose Grafana, Prometheus, and Flux UI via Tailscale LoadBalancer services

Replace Ansible port-forwarding + tailscale serve with direct Tailscale LB
services matching the existing Rancher pattern. Each service gets its own
tailnet hostname (grafana/prometheus/flux.silverside-gopher.ts.net).

ansible/site.yml

@@ -109,18 +109,6 @@
     - role: observability-content
       when: not (observability_gitops_enabled | default(true) | bool)
 
-- name: Configure private tailnet access
-  hosts: control_plane[0]
-  become: true
-  vars:
-    private_access_grafana_port: 30080
-    private_access_prometheus_port: 30990
-    private_access_flux_port: 30901
-    private_access_rancher_port: 9442
-
-  roles:
-    - private-access
-
 - name: Bootstrap Doppler access for External Secrets
   hosts: control_plane[0]
   become: true

infrastructure/addons/flux-ui/flux-tailscale-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flux-tailscale
+  namespace: flux-system
+  annotations:
+    tailscale.com/hostname: flux
+    tailscale.com/proxy-class: infra-stable
+spec:
+  type: LoadBalancer
+  loadBalancerClass: tailscale
+  selector:
+    app.kubernetes.io/name: weave-gitops
+  ports:
+    - name: http
+      port: 9001
+      protocol: TCP
+      targetPort: 9001

infrastructure/addons/flux-ui/ingress-flux-ui.yaml

@@ -1,19 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: flux-ui
-  namespace: flux-system
-  annotations:
-    traefik.ingress.kubernetes.io/router.entrypoints: flux
-spec:
-  ingressClassName: traefik
-  rules:
-    - http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: flux-system-weave-gitops
-                port:
-                  number: 9001

infrastructure/addons/flux-ui/kustomization.yaml

@@ -4,4 +4,4 @@ resources:
   - cluster-user-auth-externalsecret.yaml
   - gitrepository-weave-gitops.yaml
   - helmrelease-weave-gitops.yaml
-  - ingress-flux-ui.yaml
+  - flux-tailscale-service.yaml

infrastructure/addons/kustomization-flux-ui.yaml

@@ -12,6 +12,8 @@ spec:
   path: ./infrastructure/addons/flux-ui
   dependsOn:
     - name: addon-external-secrets
+    - name: addon-tailscale-operator
+    - name: addon-tailscale-proxyclass
   wait: true
   timeout: 5m
   suspend: false

infrastructure/addons/kustomization-observability.yaml

@@ -12,6 +12,8 @@ spec:
   path: ./infrastructure/addons/observability
   dependsOn:
     - name: addon-external-secrets
+    - name: addon-tailscale-operator
+    - name: addon-tailscale-proxyclass
   wait: true
   timeout: 5m
   suspend: false

infrastructure/addons/observability/grafana-tailscale-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-tailscale
+  namespace: observability
+  annotations:
+    tailscale.com/hostname: grafana
+    tailscale.com/proxy-class: infra-stable
+spec:
+  type: LoadBalancer
+  loadBalancerClass: tailscale
+  selector:
+    app.kubernetes.io/name: grafana
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 3000

infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml

@@ -29,8 +29,8 @@ spec:
         password: admin123
       grafana.ini:
         server:
-          root_url: http://observability/grafana/
-          serve_from_sub_path: true
+          root_url: http://grafana.silverside-gopher.ts.net/
+          serve_from_sub_path: false
       persistence:
         enabled: true
         storageClassName: local-path
@@ -50,8 +50,8 @@ spec:
       service:
         type: ClusterIP
       prometheusSpec:
-        externalUrl: http://observability/prometheus/
-        routePrefix: /prometheus/
+        externalUrl: http://prometheus.silverside-gopher.ts.net/
+        routePrefix: /
         retention: 7d
         storageSpec:
           volumeClaimTemplate:

infrastructure/addons/observability/kustomization.yaml

@@ -2,12 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  # - grafana-admin-externalsecret.yaml  # Using static password for stable baseline
-  # - traefik-tailscale-service.yaml  # Deferred - complex dependency on tailscale operator
-  # - grafana-ingress.yaml  # Deferred - requires tailscale service
-  # - prometheus-ingress.yaml  # Deferred - requires tailscale service
   - helmrepository-prometheus-community.yaml
   - helmrepository-grafana.yaml
   - helmrelease-kube-prometheus-stack.yaml
   - helmrelease-loki.yaml
   - helmrelease-promtail.yaml
+  - grafana-tailscale-service.yaml
+  - prometheus-tailscale-service.yaml

infrastructure/addons/observability/prometheus-tailscale-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-tailscale
+  namespace: observability
+  annotations:
+    tailscale.com/hostname: prometheus
+    tailscale.com/proxy-class: infra-stable
+spec:
+  type: LoadBalancer
+  loadBalancerClass: tailscale
+  selector:
+    app.kubernetes.io/name: prometheus
+  ports:
+    - name: http
+      port: 9090
+      protocol: TCP
+      targetPort: 9090