diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 7be0b12..8bf6e68 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -419,7 +419,8 @@ jobs: ghcr.io/fluxcd/notification-controller:v1.8.1 \ oci.external-secrets.io/external-secrets/external-secrets:v2.1.0 \ ghcr.io/tailscale/k8s-operator:v1.96.5 \ - ghcr.io/tailscale/tailscale:v1.96.5; do + ghcr.io/tailscale/tailscale:v1.96.5 \ + registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2; do prepare_image_archive "${image}" done @@ -724,8 +725,10 @@ jobs: import_required_image ghcr.io/tailscale/tailscale:v1.96.5 "${PRIMARY_CP_IP}" wait_for_flux_helm_release tailscale flux-system-tailscale-operator tailscale-operator tailscale-system 600s 900s 900 kubectl -n tailscale-system rollout status deployment/operator --timeout=600s - wait_for_flux_helm_release nfs-subdir-external-provisioner flux-system-nfs-subdir-external-provisioner nfs-subdir-external-provisioner kube-system 600s 600s 600 - kubectl -n kube-system rollout status deployment/kube-system-nfs-subdir-external-provisioner --timeout=600s + import_required_image registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2 "${PRIMARY_CP_IP}" + kubectl -n flux-system annotate kustomization/addon-nfs-storage reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite + kubectl -n flux-system wait --for=condition=Ready kustomization/addon-nfs-storage --timeout=300s + kubectl -n kube-system rollout status deployment/nfs-subdir-external-provisioner --timeout=300s kubectl annotate storageclass local-path storageclass.kubernetes.io/is-default-class=false --overwrite kubectl annotate storageclass flash-nfs storageclass.kubernetes.io/is-default-class=true --overwrite kubectl get storageclass flash-nfs diff --git a/infrastructure/addons/kustomization-nfs-storage.yaml b/infrastructure/addons/kustomization-nfs-storage.yaml index 67e7a1e..2402fd2 100644 --- a/infrastructure/addons/kustomization-nfs-storage.yaml +++ b/infrastructure/addons/kustomization-nfs-storage.yaml @@ -10,11 +10,11 @@ spec: kind: GitRepository name: platform path: ./infrastructure/addons/nfs-storage - wait: false + wait: true healthChecks: - - apiVersion: helm.toolkit.fluxcd.io/v2 - kind: HelmRelease + - apiVersion: apps/v1 + kind: Deployment name: nfs-subdir-external-provisioner - namespace: flux-system + namespace: kube-system timeout: 10m suspend: false diff --git a/infrastructure/addons/nfs-storage/clusterrole-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/clusterrole-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..bda91ca --- /dev/null +++ b/infrastructure/addons/nfs-storage/clusterrole-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nfs-subdir-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] diff --git a/infrastructure/addons/nfs-storage/clusterrolebinding-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/clusterrolebinding-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..3c9da2d --- /dev/null +++ b/infrastructure/addons/nfs-storage/clusterrolebinding-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: run-nfs-subdir-external-provisioner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nfs-subdir-external-provisioner-runner +subjects: + - kind: ServiceAccount + name: nfs-subdir-external-provisioner + namespace: kube-system diff --git a/infrastructure/addons/nfs-storage/deployment-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/deployment-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..d6fad7c --- /dev/null +++ b/infrastructure/addons/nfs-storage/deployment-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nfs-subdir-external-provisioner + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: nfs-subdir-external-provisioner + template: + metadata: + labels: + app: nfs-subdir-external-provisioner + spec: + serviceAccountName: nfs-subdir-external-provisioner + nodeSelector: + kubernetes.io/hostname: k8s-cluster-cp-1 + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: nfs-subdir-external-provisioner + image: registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2 + imagePullPolicy: IfNotPresent + env: + - name: PROVISIONER_NAME + value: flash-nfs + - name: NFS_SERVER + value: 10.27.27.239 + - name: NFS_PATH + value: /TheFlash/k8s-nfs + volumeMounts: + - name: nfs-subdir-external-provisioner-root + mountPath: /persistentvolumes + volumes: + - name: nfs-subdir-external-provisioner-root + nfs: + server: 10.27.27.239 + path: /TheFlash/k8s-nfs diff --git a/infrastructure/addons/nfs-storage/helmrelease-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/helmrelease-nfs-subdir-external-provisioner.yaml deleted file mode 100644 index 9ed1699..0000000 --- a/infrastructure/addons/nfs-storage/helmrelease-nfs-subdir-external-provisioner.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: nfs-subdir-external-provisioner - namespace: flux-system -spec: - interval: 10m - targetNamespace: kube-system - chart: - spec: - chart: nfs-subdir-external-provisioner - version: 4.0.18 - sourceRef: - kind: HelmRepository - name: nfs-subdir-external-provisioner - namespace: flux-system - install: - createNamespace: true - remediation: - retries: 3 - upgrade: - remediation: - retries: 3 - values: - nfs: - server: 10.27.27.239 - path: /TheFlash/k8s-nfs - storageClass: - create: true - defaultClass: true - name: flash-nfs - provisionerName: flash-nfs - reclaimPolicy: Delete - archiveOnDelete: true - allowVolumeExpansion: true - volumeBindingMode: Immediate diff --git a/infrastructure/addons/nfs-storage/helmrepository-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/helmrepository-nfs-subdir-external-provisioner.yaml deleted file mode 100644 index 8f234ed..0000000 --- a/infrastructure/addons/nfs-storage/helmrepository-nfs-subdir-external-provisioner.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: nfs-subdir-external-provisioner - namespace: flux-system -spec: - interval: 1h - url: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner diff --git a/infrastructure/addons/nfs-storage/kustomization.yaml b/infrastructure/addons/nfs-storage/kustomization.yaml index 38eb6a5..1d4e42c 100644 --- a/infrastructure/addons/nfs-storage/kustomization.yaml +++ b/infrastructure/addons/nfs-storage/kustomization.yaml @@ -1,5 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - helmrepository-nfs-subdir-external-provisioner.yaml - - helmrelease-nfs-subdir-external-provisioner.yaml + - serviceaccount-nfs-subdir-external-provisioner.yaml + - clusterrole-nfs-subdir-external-provisioner.yaml + - clusterrolebinding-nfs-subdir-external-provisioner.yaml + - role-nfs-subdir-external-provisioner.yaml + - rolebinding-nfs-subdir-external-provisioner.yaml + - storageclass-flash-nfs.yaml + - deployment-nfs-subdir-external-provisioner.yaml diff --git a/infrastructure/addons/nfs-storage/role-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/role-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..f024644 --- /dev/null +++ b/infrastructure/addons/nfs-storage/role-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-locking-nfs-subdir-external-provisioner + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/infrastructure/addons/nfs-storage/rolebinding-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/rolebinding-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..21ce19d --- /dev/null +++ b/infrastructure/addons/nfs-storage/rolebinding-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-locking-nfs-subdir-external-provisioner + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-locking-nfs-subdir-external-provisioner +subjects: + - kind: ServiceAccount + name: nfs-subdir-external-provisioner + namespace: kube-system diff --git a/infrastructure/addons/nfs-storage/serviceaccount-nfs-subdir-external-provisioner.yaml b/infrastructure/addons/nfs-storage/serviceaccount-nfs-subdir-external-provisioner.yaml new file mode 100644 index 0000000..6a1b316 --- /dev/null +++ b/infrastructure/addons/nfs-storage/serviceaccount-nfs-subdir-external-provisioner.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfs-subdir-external-provisioner + namespace: kube-system diff --git a/infrastructure/addons/nfs-storage/storageclass-flash-nfs.yaml b/infrastructure/addons/nfs-storage/storageclass-flash-nfs.yaml new file mode 100644 index 0000000..1da6f6d --- /dev/null +++ b/infrastructure/addons/nfs-storage/storageclass-flash-nfs.yaml @@ -0,0 +1,12 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: flash-nfs + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: flash-nfs +parameters: + archiveOnDelete: "true" +reclaimPolicy: Delete +allowVolumeExpansion: true +volumeBindingMode: Immediate