diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 58c8f61..53e59d0 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -230,6 +230,7 @@ jobs: -e "tailscale_tailnet=${{ secrets.TAILSCALE_TAILNET }}" \ -e "tailscale_oauth_client_id=${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}" \ -e "tailscale_oauth_client_secret=${{ secrets.TAILSCALE_OAUTH_CLIENT_SECRET }}" \ + -e "doppler_hetznerterra_service_token=${{ secrets.DOPPLER_HETZNERTERRA_SERVICE_TOKEN }}" \ -e "grafana_admin_password=${{ secrets.GRAFANA_ADMIN_PASSWORD }}" \ -e "cluster_name=k8s-cluster" env: diff --git a/README.md b/README.md index 50dc5bd..dc7cfb1 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ Set these in your Gitea repository settings (**Settings** → **Secrets** → ** | `TAILSCALE_TAILNET` | Tailnet domain (e.g., `yourtailnet.ts.net`) | | `TAILSCALE_OAUTH_CLIENT_ID` | Tailscale OAuth client ID for Kubernetes Operator | | `TAILSCALE_OAUTH_CLIENT_SECRET` | Tailscale OAuth client secret for Kubernetes Operator | +| `DOPPLER_HETZNERTERRA_SERVICE_TOKEN` | Doppler service token for `hetznerterra` runtime secrets | | `GRAFANA_ADMIN_PASSWORD` | Optional admin password for Grafana (auto-generated if unset) | | `RUNNER_ALLOWED_CIDRS` | Optional CIDR list for CI runner access if you choose to pass it via tfvars/secrets | | `SSH_PUBLIC_KEY` | SSH public key content | @@ -178,6 +179,19 @@ Set these in your Gitea repository settings (**Settings** → **Secrets** → ** This repo now includes a Flux GitOps layout for phased migration from imperative Ansible applies to continuous reconciliation. +### Runtime secrets + +Runtime cluster secrets are moving to Doppler + External Secrets Operator. + +- Doppler project: `hetznerterra` +- Initial auth: service token via `DOPPLER_HETZNERTERRA_SERVICE_TOKEN` +- First synced secrets: + - `GRAFANA_ADMIN_PASSWORD` + - `WEAVE_GITOPS_ADMIN_USERNAME` + - `WEAVE_GITOPS_ADMIN_PASSWORD_BCRYPT_HASH` + +Terraform/bootstrap secrets remain in Gitea Actions secrets and are not managed by Doppler. + ### Repository layout - `clusters/prod/`: cluster entrypoint and Flux reconciliation objects diff --git a/SECRETS_SETUP.md b/SECRETS_SETUP.md index 2e5a2d9..f162f77 100644 --- a/SECRETS_SETUP.md +++ b/SECRETS_SETUP.md @@ -54,9 +54,14 @@ Add these secrets in your Gitea repository settings: ### Application Secrets +#### `DOPPLER_HETZNERTERRA_SERVICE_TOKEN` +- Doppler service token for the `hetznerterra` project runtime secrets +- Used by External Secrets Operator bootstrap +- Recommended scope: `hetznerterra` project, `prod` config only + #### `GRAFANA_ADMIN_PASSWORD` -- Admin password for Grafana -- Generate a strong password: `openssl rand -base64 32` +- Transitional fallback only while migrating observability secrets to Doppler +- In steady state, store this in Doppler as `GRAFANA_ADMIN_PASSWORD` ## Setting Up Secrets @@ -82,6 +87,7 @@ Check the workflow logs to verify all secrets are being used correctly. - Never commit secrets to the repository - Use strong, unique passwords for Grafana and other services +- Prefer Doppler for runtime app/platform secrets after cluster bootstrap - Rotate Tailscale auth keys periodically - Review OAuth client permissions regularly - The workflow automatically opens SSH/API access only for the runner's IP during deployment diff --git a/ansible/roles/doppler-bootstrap/tasks/main.yml b/ansible/roles/doppler-bootstrap/tasks/main.yml new file mode 100644 index 0000000..d52c063 --- /dev/null +++ b/ansible/roles/doppler-bootstrap/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Ensure Doppler service token is provided + assert: + that: + - doppler_hetznerterra_service_token | length > 0 + fail_msg: doppler_hetznerterra_service_token must be provided for External Secrets bootstrap. + +- name: Ensure external-secrets namespace exists + shell: kubectl create namespace external-secrets --dry-run=client -o yaml | kubectl apply -f - + changed_when: true + +- name: Apply Doppler service token secret + shell: >- + kubectl -n external-secrets create secret generic doppler-hetznerterra-service-token + --from-literal=dopplerToken='{{ doppler_hetznerterra_service_token }}' + --dry-run=client -o yaml | kubectl apply -f - + changed_when: true diff --git a/ansible/site.yml b/ansible/site.yml index 3cffcf5..a504517 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -123,6 +123,13 @@ roles: - private-access +- name: Bootstrap Doppler access for External Secrets + hosts: control_plane[0] + become: true + + roles: + - doppler-bootstrap + - name: Finalize hosts: localhost connection: local diff --git a/infrastructure/addons/external-secrets/clustersecretstore-doppler-hetznerterra.yaml b/infrastructure/addons/external-secrets/clustersecretstore-doppler-hetznerterra.yaml new file mode 100644 index 0000000..dcb6068 --- /dev/null +++ b/infrastructure/addons/external-secrets/clustersecretstore-doppler-hetznerterra.yaml @@ -0,0 +1,13 @@ +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: doppler-hetznerterra +spec: + provider: + doppler: + auth: + secretRef: + dopplerToken: + name: doppler-hetznerterra-service-token + key: dopplerToken + namespace: external-secrets diff --git a/infrastructure/addons/external-secrets/helmrelease-external-secrets.yaml b/infrastructure/addons/external-secrets/helmrelease-external-secrets.yaml new file mode 100644 index 0000000..9cfcef8 --- /dev/null +++ b/infrastructure/addons/external-secrets/helmrelease-external-secrets.yaml @@ -0,0 +1,27 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: external-secrets + namespace: flux-system +spec: + interval: 10m + targetNamespace: external-secrets + chart: + spec: + chart: external-secrets + version: 2.1.0 + sourceRef: + kind: HelmRepository + name: external-secrets + namespace: flux-system + install: + createNamespace: true + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + installCRDs: true + serviceMonitor: + enabled: false diff --git a/infrastructure/addons/external-secrets/helmrepository-external-secrets.yaml b/infrastructure/addons/external-secrets/helmrepository-external-secrets.yaml new file mode 100644 index 0000000..1128f31 --- /dev/null +++ b/infrastructure/addons/external-secrets/helmrepository-external-secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: external-secrets + namespace: flux-system +spec: + interval: 1h + url: https://charts.external-secrets.io diff --git a/infrastructure/addons/external-secrets/kustomization.yaml b/infrastructure/addons/external-secrets/kustomization.yaml new file mode 100644 index 0000000..75d409c --- /dev/null +++ b/infrastructure/addons/external-secrets/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - helmrepository-external-secrets.yaml + - helmrelease-external-secrets.yaml + - clustersecretstore-doppler-hetznerterra.yaml diff --git a/infrastructure/addons/external-secrets/namespace.yaml b/infrastructure/addons/external-secrets/namespace.yaml new file mode 100644 index 0000000..4ef398e --- /dev/null +++ b/infrastructure/addons/external-secrets/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-secrets diff --git a/infrastructure/addons/flux-ui/cluster-user-auth-externalsecret.yaml b/infrastructure/addons/flux-ui/cluster-user-auth-externalsecret.yaml new file mode 100644 index 0000000..0ae9723 --- /dev/null +++ b/infrastructure/addons/flux-ui/cluster-user-auth-externalsecret.yaml @@ -0,0 +1,25 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: cluster-user-auth + namespace: flux-system +spec: + refreshInterval: 1h + secretStoreRef: + name: doppler-hetznerterra + kind: ClusterSecretStore + target: + name: cluster-user-auth + creationPolicy: Owner + template: + type: Opaque + data: + username: "{{ .fluxAdminUsername }}" + password: "{{ .fluxAdminPasswordHash }}" + data: + - secretKey: fluxAdminUsername + remoteRef: + key: WEAVE_GITOPS_ADMIN_USERNAME + - secretKey: fluxAdminPasswordHash + remoteRef: + key: WEAVE_GITOPS_ADMIN_PASSWORD_BCRYPT_HASH diff --git a/infrastructure/addons/flux-ui/helmrelease-weave-gitops.yaml b/infrastructure/addons/flux-ui/helmrelease-weave-gitops.yaml index 83a4104..579a332 100644 --- a/infrastructure/addons/flux-ui/helmrelease-weave-gitops.yaml +++ b/infrastructure/addons/flux-ui/helmrelease-weave-gitops.yaml @@ -27,9 +27,8 @@ spec: adminUser: create: true createClusterRole: true - createSecret: true + createSecret: false username: admin - passwordHash: "$2b$12$iVSpwZxP98Y1T4AOwj.TAeMsrOuQ6vWfhXfG4Gan9ay.qGMaRNdrC" rbac: create: true impersonationResourceNames: diff --git a/infrastructure/addons/flux-ui/kustomization.yaml b/infrastructure/addons/flux-ui/kustomization.yaml index 778b2b6..b3bd793 100644 --- a/infrastructure/addons/flux-ui/kustomization.yaml +++ b/infrastructure/addons/flux-ui/kustomization.yaml @@ -1,6 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - cluster-user-auth-externalsecret.yaml - gitrepository-weave-gitops.yaml - helmrelease-weave-gitops.yaml - traefik-helmchartconfig-flux-entrypoint.yaml diff --git a/infrastructure/addons/kustomization-external-secrets.yaml b/infrastructure/addons/kustomization-external-secrets.yaml new file mode 100644 index 0000000..cd82aed --- /dev/null +++ b/infrastructure/addons/kustomization-external-secrets.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: addon-external-secrets + namespace: flux-system +spec: + interval: 10m + prune: true + sourceRef: + kind: GitRepository + name: platform + path: ./infrastructure/addons/external-secrets + wait: true + timeout: 5m + suspend: false diff --git a/infrastructure/addons/kustomization-flux-ui.yaml b/infrastructure/addons/kustomization-flux-ui.yaml index 5c23ff1..729ef5c 100644 --- a/infrastructure/addons/kustomization-flux-ui.yaml +++ b/infrastructure/addons/kustomization-flux-ui.yaml @@ -10,6 +10,8 @@ spec: kind: GitRepository name: platform path: ./infrastructure/addons/flux-ui + dependsOn: + - name: addon-external-secrets wait: true timeout: 5m suspend: false diff --git a/infrastructure/addons/kustomization-observability.yaml b/infrastructure/addons/kustomization-observability.yaml index 17ce3da..877d5b6 100644 --- a/infrastructure/addons/kustomization-observability.yaml +++ b/infrastructure/addons/kustomization-observability.yaml @@ -10,6 +10,8 @@ spec: kind: GitRepository name: platform path: ./infrastructure/addons/observability + dependsOn: + - name: addon-external-secrets wait: true timeout: 5m suspend: false diff --git a/infrastructure/addons/kustomization.yaml b/infrastructure/addons/kustomization.yaml index 7bb5ec6..d692f0c 100644 --- a/infrastructure/addons/kustomization.yaml +++ b/infrastructure/addons/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - kustomization-ccm.yaml - kustomization-csi.yaml + - kustomization-external-secrets.yaml - kustomization-flux-ui.yaml - kustomization-tailscale-operator.yaml - kustomization-observability.yaml diff --git a/infrastructure/addons/observability/grafana-admin-externalsecret.yaml b/infrastructure/addons/observability/grafana-admin-externalsecret.yaml new file mode 100644 index 0000000..3dd5ab3 --- /dev/null +++ b/infrastructure/addons/observability/grafana-admin-externalsecret.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: grafana-admin + namespace: observability +spec: + refreshInterval: 1h + secretStoreRef: + name: doppler-hetznerterra + kind: ClusterSecretStore + target: + name: grafana-admin-credentials + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: admin + admin-password: "{{ .grafanaAdminPassword }}" + data: + - secretKey: grafanaAdminPassword + remoteRef: + key: GRAFANA_ADMIN_PASSWORD diff --git a/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml b/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml index 1c103aa..17729c4 100644 --- a/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml +++ b/infrastructure/addons/observability/helmrelease-kube-prometheus-stack.yaml @@ -24,6 +24,10 @@ spec: values: grafana: enabled: true + admin: + existingSecret: grafana-admin-credentials + userKey: admin-user + passwordKey: admin-password grafana.ini: server: root_url: http://observability/grafana/ diff --git a/infrastructure/addons/observability/kustomization.yaml b/infrastructure/addons/observability/kustomization.yaml index b9c74f0..a90a64e 100644 --- a/infrastructure/addons/observability/kustomization.yaml +++ b/infrastructure/addons/observability/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace.yaml + - grafana-admin-externalsecret.yaml - traefik-tailscale-service.yaml - grafana-ingress.yaml - prometheus-ingress.yaml