From 4c104f74e8b4c6d5d8017fad72cedd61cb3f6755 Mon Sep 17 00:00:00 2001
From: MichaelFisher1997 <contact@michaelfisher.tech>
Date: Sat, 7 Mar 2026 01:04:03 +0000
Subject: [PATCH] feat: route observability through one tailscale endpoint

---
 README.md                                     |  6 ++---
 .../addons/observability/grafana-ingress.yaml | 17 ++++++++++++++
 .../helmrelease-kube-prometheus-stack.yaml    | 18 +++++++--------
 .../addons/observability/kustomization.yaml   |  3 +++
 .../observability/prometheus-ingress.yaml     | 17 ++++++++++++++
 .../traefik-tailscale-service.yaml            | 23 +++++++++++++++++++
 6 files changed, 71 insertions(+), 13 deletions(-)
 create mode 100644 infrastructure/addons/observability/grafana-ingress.yaml
 create mode 100644 infrastructure/addons/observability/prometheus-ingress.yaml
 create mode 100644 infrastructure/addons/observability/traefik-tailscale-service.yaml

diff --git a/README.md b/README.md
index 7e05130..94816b1 100644
--- a/README.md
+++ b/README.md
@@ -225,14 +225,14 @@ Flux deploys a lightweight observability stack in the `observability` namespace:
 
 Grafana content is managed as code via ConfigMaps in `infrastructure/addons/observability-content/` (Flux), migrated from `ansible/roles/observability-content/`.
 
-Grafana and Prometheus are exposed via Tailscale (`loadBalancerClass: tailscale`) when the Tailscale Kubernetes Operator is healthy.
+Grafana and Prometheus are exposed through a single Tailscale front door backed by Traefik when the Tailscale Kubernetes Operator is healthy.
 
 ### Access Grafana and Prometheus
 
 Preferred (when Tailscale Operator is healthy):
 
-- Grafana: `http://grafana` (or `http://grafana.<your-tailnet>`)
-- Prometheus: `http://prometheus` (or `http://prometheus.<your-tailnet>`)
+- Grafana: `http://observability/grafana/` (or `http://observability.<your-tailnet>/grafana/`)
+- Prometheus: `http://observability/prometheus/` (or `http://observability.<your-tailnet>/prometheus/`)
 
 Fallback (port-forward from a tailnet-connected machine):
 
diff --git a/infrastructure/addons/observability/grafana-ingress.yaml b/infrastructure/addons/observability/grafana-ingress.yaml
new file mode 100644
index 0000000..7156c5d
--- /dev/null
+++ b/infrastructure/addons/observability/grafana-ingress.yaml
@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: observability
+spec:
+  ingressClassName: traefik
+  rules:
+    - http:
+        paths:
+          - path: /grafana
+            pathType: Prefix
+            backend:
+              service:
+                name: observability-kube-prometheus-stack-grafana
+                port:
+                  number: 80
diff --git a/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml b/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml
index b0931c1..4629b0b 100644
--- a/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml
+++ b/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml
@@ -24,16 +24,16 @@ spec:
   values:
     grafana:
       enabled: true
+      grafana.ini:
+        server:
+          root_url: http://observability/grafana/
+          serve_from_sub_path: true
       persistence:
         enabled: true
         storageClassName: local-path
         size: 5Gi
       service:
-        type: LoadBalancer
-        loadBalancerClass: tailscale
-        annotations:
-          tailscale.com/hostname: grafana
-          tailscale.com/proxy-class: infra-stable
+        type: ClusterIP
       sidecar:
         datasources:
           enabled: true
@@ -45,12 +45,10 @@ spec:
           searchNamespace: observability
     prometheus:
       service:
-        type: LoadBalancer
-        loadBalancerClass: tailscale
-        annotations:
-          tailscale.com/hostname: prometheus
-          tailscale.com/proxy-class: infra-stable
+        type: ClusterIP
       prometheusSpec:
+        externalUrl: http://observability/prometheus/
+        routePrefix: /prometheus/
         retention: 7d
         storageSpec:
           volumeClaimTemplate:
diff --git a/infrastructure/addons/observability/kustomization.yaml b/infrastructure/addons/observability/kustomization.yaml
index 52be631..b9c74f0 100644
--- a/infrastructure/addons/observability/kustomization.yaml
+++ b/infrastructure/addons/observability/kustomization.yaml
@@ -2,6 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
+  - traefik-tailscale-service.yaml
+  - grafana-ingress.yaml
+  - prometheus-ingress.yaml
   - helmrepository-prometheus-community.yaml
   - helmrepository-grafana.yaml
   - helmrelease-kube-prometheus-stack.yaml
diff --git a/infrastructure/addons/observability/prometheus-ingress.yaml b/infrastructure/addons/observability/prometheus-ingress.yaml
new file mode 100644
index 0000000..3251184
--- /dev/null
+++ b/infrastructure/addons/observability/prometheus-ingress.yaml
@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus
+  namespace: observability
+spec:
+  ingressClassName: traefik
+  rules:
+    - http:
+        paths:
+          - path: /prometheus
+            pathType: Prefix
+            backend:
+              service:
+                name: observability-kube-prometh-prometheus
+                port:
+                  number: 9090
diff --git a/infrastructure/addons/observability/traefik-tailscale-service.yaml b/infrastructure/addons/observability/traefik-tailscale-service.yaml
new file mode 100644
index 0000000..a841771
--- /dev/null
+++ b/infrastructure/addons/observability/traefik-tailscale-service.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-tailscale
+  namespace: kube-system
+  annotations:
+    tailscale.com/hostname: observability
+    tailscale.com/proxy-class: infra-stable
+spec:
+  type: LoadBalancer
+  loadBalancerClass: tailscale
+  selector:
+    app.kubernetes.io/instance: traefik-kube-system
+    app.kubernetes.io/name: traefik
+  ports:
+    - name: web
+      port: 80
+      protocol: TCP
+      targetPort: web
+    - name: websecure
+      port: 443
+      protocol: TCP
+      targetPort: websecure