Add production observability manifests

deploy/prod/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - webshare-api-externalsecret.yaml
   - site-analyzer-deployment.yaml
   - site-analyzer-service.yaml
+  - site-analyzer-servicemonitor.yaml
   - site-analyzer-hpa.yaml
   - lightpanda-cdp-deployment.yaml
   - lightpanda-cdp-service.yaml
@@ -14,7 +15,10 @@ resources:
   - lightpanda-mcp-hpa.yaml
   - scraper-deployment.yaml
   - scraper-service.yaml
+  - scraper-servicemonitor.yaml
   - scraper-hpa.yaml
+  - microservices-workload-prometheusrule.yaml
+  - microservices-http-prometheusrule.yaml
   - traefik-middleware-strip-prefix.yaml
   - ingressroute-microservices.yaml
 images:

deploy/prod/lightpanda-cdp-service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: lightpanda-cdp
   namespace: microservices
+  labels:
+    app: lightpanda-cdp
 spec:
   type: ClusterIP
   selector:

deploy/prod/lightpanda-mcp-service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: lightpanda-mcp
   namespace: microservices
+  labels:
+    app: lightpanda-mcp
 spec:
   type: ClusterIP
   selector:

deploy/prod/microservices-http-prometheusrule.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: microservices-http
+  namespace: microservices
+  labels:
+    app: microservices
+spec:
+  groups:
+    - name: microservices.http
+      rules:
+        - alert: MicroserviceHigh5xxRate
+          expr: sum by (service) (rate(http_requests_total{service=~"scraper|site-analyzer",status=~"5.."}[5m])) / clamp_min(sum by (service) (rate(http_requests_total{service=~"scraper|site-analyzer"}[5m])), 0.001) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Microservice has elevated 5xx errors"
+            description: "{{ $labels.service }} has more than 5% 5xx responses over 10 minutes."
+        - alert: MicroserviceHighP95Latency
+          expr: histogram_quantile(0.95, sum by (service, le) (rate(http_request_duration_seconds_bucket{service=~"scraper|site-analyzer",path!~"/health|/ready"}[5m]))) > 2
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Microservice has high p95 latency"
+            description: "{{ $labels.service }} p95 HTTP latency is above 2 seconds over 10 minutes."

deploy/prod/microservices-workload-prometheusrule.yaml

@@ -0,0 +1,27 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: microservices-workloads
+  namespace: microservices
+  labels:
+    app: microservices
+spec:
+  groups:
+    - name: microservices.workloads
+      rules:
+        - alert: MicroserviceDeploymentUnavailable
+          expr: kube_deployment_status_replicas_unavailable{namespace="microservices",deployment=~"scraper|site-analyzer|lightpanda-cdp|lightpanda-mcp"} > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Microservice deployment has unavailable replicas"
+            description: "{{ $labels.deployment }} has {{ $value }} unavailable replica(s) in namespace {{ $labels.namespace }}."
+        - alert: MicroservicePodRestarting
+          expr: increase(kube_pod_container_status_restarts_total{namespace="microservices",container=~"scraper|site-analyzer|lightpanda-cdp|lightpanda-mcp"}[15m]) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Microservice pod is restarting"
+            description: "{{ $labels.pod }}/{{ $labels.container }} restarted {{ $value }} times in the last 15 minutes."

deploy/prod/scraper-deployment.yaml

@@ -56,7 +56,7 @@ spec:
             periodSeconds: 10
           readinessProbe:
             httpGet:
-              path: /health
+              path: /ready
               port: 8080
             initialDelaySeconds: 2
             periodSeconds: 5

deploy/prod/scraper-service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: scraper
   namespace: microservices
+  labels:
+    app: scraper
 spec:
   type: ClusterIP
   selector:

deploy/prod/scraper-servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: scraper
+  namespace: microservices
+  labels:
+    app: scraper
+spec:
+  selector:
+    matchLabels:
+      app: scraper
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

deploy/prod/site-analyzer-service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: site-analyzer
   namespace: microservices
+  labels:
+    app: site-analyzer
 spec:
   type: ClusterIP
   selector:

deploy/prod/site-analyzer-servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: site-analyzer
+  namespace: microservices
+  labels:
+    app: site-analyzer
+spec:
+  selector:
+    matchLabels:
+      app: site-analyzer
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s