From d14ee6ba1d1cf62ee9ca766bc38fb77eacc95333 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Mon, 27 May 2024 16:22:56 +0200 Subject: [PATCH] helm: update AWS CSI driver chart (#3121) --- internal/constellation/helm/BUILD.bazel | 5 + .../helm/charts/edgeless/csi/Chart.yaml | 2 +- .../csi/charts/aws-csi-driver/CHANGELOG.md | 116 ++++++++ .../csi/charts/aws-csi-driver/Chart.yaml | 4 +- .../charts/aws-csi-driver/templates/NOTES.txt | 2 +- .../templates/_node-windows.tpl | 262 ++++++++++++++++++ .../charts/aws-csi-driver/templates/_node.tpl | 250 +++++++++++++++++ .../templates/clusterrole-attacher.yaml | 3 + .../templates/clusterrole-csi-node.yaml | 6 + .../templates/clusterrole-provisioner.yaml | 9 +- .../templates/clusterrole-resizer.yaml | 6 + .../templates/clusterrole-snapshotter.yaml | 5 +- .../aws-csi-driver/templates/controller.yaml | 186 +++++++++++-- .../templates/ebs-csi-default-sc.yaml | 11 + .../aws-csi-driver/templates/metrics.yaml | 2 +- .../templates/node-windows.yaml | 195 +------------ .../charts/aws-csi-driver/templates/node.yaml | 252 +++-------------- .../poddisruptionbudget-controller.yaml | 4 - .../aws-csi-driver/templates/role-leases.yaml | 11 + .../templates/rolebinding-leases.yaml | 15 + .../serviceaccount-csi-controller.yaml | 3 + .../templates/serviceaccount-csi-node.yaml | 3 + .../templates/volumesnapshotclass.yaml | 3 + .../csi/charts/aws-csi-driver/values.yaml | 184 +++++++++++- .../constellation/helm/update-csi-charts.sh | 2 +- 25 files changed, 1104 insertions(+), 437 deletions(-) create mode 100644 internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl create mode 100644 internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl create mode 100644 internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml create mode 100644 internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml create mode 100644 internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml diff --git a/internal/constellation/helm/BUILD.bazel b/internal/constellation/helm/BUILD.bazel index 6e3c5eee7..52f92729c 100644 --- a/internal/constellation/helm/BUILD.bazel +++ b/internal/constellation/helm/BUILD.bazel @@ -457,6 +457,11 @@ go_library( "charts/yawol/charts/yawol-controller/values.yaml", "charts/yawol/templates/.gitkeep", "charts/yawol/values.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl", + "charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl", + "charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml", ], importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm", visibility = ["//:__subpackages__"], diff --git a/internal/constellation/helm/charts/edgeless/csi/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/Chart.yaml index 3fb20c87e..3a1feaf37 100644 --- a/internal/constellation/helm/charts/edgeless/csi/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/Chart.yaml @@ -9,7 +9,7 @@ dependencies: - name: snapshot-crds version: 6.2.2 - name: aws-csi-driver - version: 1.1.0 + version: 1.2.0 tags: - AWS - name: azuredisk-csi-driver diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md index bc6aa0036..3daf61e5e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md @@ -1,4 +1,120 @@ # Helm chart +## v2.30.0 +* Bump driver version to `v1.30.0` +* Update voluemessnapshotcontents/status RBAC ([#1991](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1991), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Upgrade dependencies ([#2016](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2016), [@torredil](https://github.com/torredil)) + +## v2.29.1 +* Bump driver version to `v1.29.1` +* Remove `--reuse-values` deprecation warning + +## v2.29.0 +### Urgent Upgrade Notes +*(No, really, you MUST read this before you upgrade)* + +The EBS CSI Driver Helm chart no longer supports upgrading with `--reuse-values`. This chart will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`. + +For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864). + +### Other Changes +* Bump driver version to `v1.29.0` and sidecars to latest versions +* Add helm-tester enabled flag ([#1954](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1954), [@nunodomingues-td](https://github.com/nunodomingues-td)) + +## v2.28.1 +* Add `reservedVolumeAttachments` that overrides heuristic-determined reserved attachments via `--reserved-volume-attachments` CLI option from [PR #1919](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1919) through Helm ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add `additionalArgs` parameter to node daemonSet ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.28.0 +### Urgent Upgrade Notes +*(No, really, you MUST read this before you upgrade)* + +This is the last minor version of the EBS CSI Driver Helm chart to support upgrading with `--reuse-values`. Future versions of the chart (starting with `v2.29.0`) will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`. + +For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864). + +### Other Changes +* Bump driver version to `v1.28.0` and sidecars to latest versions +* Add labels to leases role used by EBS CSI controller ([#1914](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1914), [@cHiv0rz](https://github.com/cHiv0rz)) +* Enforce `linux` and `amd64` node affinity for helm tester pod ([#1922](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1922), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add configuration for `DaemonSet` annotations ([#1923](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1923), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Incorporate KubeLinter recommended best practices for chart tester pod ([#1924](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1924), [@torredil](https://github.com/torredil)) +* Add configuration for chart tester pod image ([#1928](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1928), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.27.0 +* Bump driver version to `v1.27.0` +* Add parameters for tuning revisionHistoryLimit and emptyDir volumes ([#1840](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1840), [@bodgit](https://github.com/bodgit)) + +## v2.26.1 +* Bump driver version to `v1.26.1` +* Bump sidecar container versions to fix [restart bug in external attacher, provisioner, resizer, snapshotter, and node-driver-registrar](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1875) ([#1886](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1886), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.26.0 +* Bump driver version to `v1.26.0` +* Bump sidecar container versions ([#1867](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1867), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add warning about --reuse-values deprecation to NOTES.txt ([#1865](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1865), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.25.0 +* Bump driver version to `v1.25.0` +* Update default sidecar timeout values ([#1824](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1824), [@torredil](https://github.com/torredil)) +* Increase default QPS and worker threads of sidecars ([#1834](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1834), [@ConnorJC3](https://github.com/ConnorJC3)) +* Node-driver-registrar sidecar fixes ([#1815](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1815), [@jukie](https://github.com/jukie)) +* Suggest eks.amazonaws.com/role-arn in values.yaml if EKS IAM for SA is used ([#1804](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1804), [@tporeba](https://github.com/tporeba)) + +## v2.24.1 +* Bump driver version to `v1.24.1` +* Upgrade sidecar images + +## v2.24.0 +* Bump driver version to `v1.24.0` +* Add additionalClusterRoleRules to sidecar chart templates. ([#1757](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1757), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Allow passing template value for clusterName ([#1753](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1753), [@monicastanciu](https://github.com/monicastanciu)) +* Make hostNetwork configurable for daemonset ([#1716](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1716), [@bseenu](https://github.com/bseenu)) +* Add labels to volumesnapshotclass ([#1754](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1754), [@fad3t](https://github.com/fad3t)) +* Update default API version for PodDisruptionBudget ([#1751](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1751), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.23.2 +* Bump driver version to `v1.23.2` +* Upgrade sidecar images + +## v2.23.1 +* Bump driver version to `v1.23.1` + +## v2.23.0 +* Add `node.enableLinux` parameter ([#1732](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1732), [@monicastanciu](https://github.com/monicastanciu)) +* Additional Node DaemonSets bug fixes ([#1739](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1739), [@monicastanciu](https://github.com/monicastanciu)) +* Additional DaemonSets feature ([#1722](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1722), [@ConnorJC3](https://github.com/ConnorJC3)) +* Add doc of chart value additionalArgs ([#1697](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1697), [@zitudu](https://github.com/zitudu)) + +## v2.22.1 +* Bump driver version to `v1.22.1` + +## v2.22.0 +* Default PodDisruptionBudget to policy/v1 ([#1707](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1707), [@iNoahNothing](https://github.com/iNoahNothing)) + +## v2.21.0 +* Bump driver version to `v1.21.0` +* Enable additional volume mounts on node pods ([#1670](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1670), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Enable customization of aws-secret name and keys in Helm Chart ([#1668](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1668), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* The sidecars have been updated. The new versions are: + - csi-snapshotter: `v6.2.2` + +## v2.20.0 +* Bump driver version to `v1.20.0` +* Enable leader election in csi-resizer sidecar ([#1606](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1606), [@rdpsin](https://github.com/rdpsin)) +* Namespace-scoped leases permissions ([#1614](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1614), [@torredil](https://github.com/torredil)) +* Add additionalArgs parameter for sidecars ([#1627](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1627), [@ConnorJC3](https://github.com/ConnorJC3)) +* Avoid generating manifests with empty envFrom fields ([#1630](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1630), [@mvgmb](https://github.com/mvgmb)) +* Allow to set automountServiceAccountToken in ServiceAccount ([#1619](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1619), [@kahirokunn](https://github.com/kahirokunn)) + +## v2.19.0 +* Bump driver version to `v1.19.0` +* The sidecars have been updated. The new versions are: + - csi-provisioner: `v3.5.0` + - csi-attacher: `v4.3.0` + - livenessprobe: `v2.10.0` + - csi-resizer: `v1.8.0` + - node-driver-registrar: `v2.8.0` +* Remove CPU limits ([#1596](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1596), [@torredil](https://github.com/torredil)) ## v2.18.0 ### Urgent Upgrade Notes diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml index fc4e85297..c439f3ef6 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" description: AWS Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: aws-csi-driver kubeVersion: ">=1.17.0-0" diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt index 5d79084ec..cb3e6cecf 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt @@ -2,4 +2,4 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. \ No newline at end of file +NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl new file mode 100644 index 000000000..ab17f71e5 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl @@ -0,0 +1,262 @@ +{{- define "node-windows" }} +{{- if .Values.node.enableWindows }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ printf "%s-windows" .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{ toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.node.windowsHostProcess }} + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "aws-ebs-csi-driver.exe" + {{- end }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + {{- if .Values.node.windowsHostProcess }} + - --windows-host-process=true + {{- end }} + env: + - name: CSI_ENDPOINT + {{- if .Values.node.windowsHostProcess }} + value: unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: unix:/csi/csi.sock + {{- end }} + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + {{- if not .Values.node.windowsHostProcess }} + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if not .Values.node.windowsHostProcess }} + securityContext: + windowsOptions: + runAsUserName: "ContainerAdministrator" + {{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "csi-node-driver-registrar.exe" + {{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + {{- if .Values.node.windowsHostProcess }} + - --plugin-registration-path=$(PLUGIN_REG_DIR) + {{- end }} + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + {{- if .Values.node.windowsHostProcess }} + value: unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: unix:/csi/csi.sock + {{- end }} + - name: DRIVER_REG_SOCK_PATH + {{- if .Values.node.windowsHostProcess }} + value: C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock + {{- end }} + {{- if .Values.node.windowsHostProcess }} + - name: PLUGIN_REG_DIR + value: C:\\var\\lib\\kubelet\\plugins_registry\\ + {{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + periodSeconds: 90 + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + - name: probe-dir + mountPath: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "livenessprobe.exe" + {{- end }} + args: + {{- if .Values.node.windowsHostProcess }} + - --csi-address=unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + - --csi-address=unix:/csi/csi.sock + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + {{- if not .Values.node.windowsHostProcess }} + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + {{- end }} + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl new file mode 100644 index 000000000..4591f7efe --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl @@ -0,0 +1,250 @@ +{{- define "node" }} +{{- if or (eq (default true .Values.node.enableLinux) true) }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.daemonSetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + - key: "ebs.csi.aws.com/agent-not-ready" + operator: "Exists" + {{- end }} + hostNetwork: {{ .Values.node.hostNetwork }} + {{- with .Values.node.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.reservedVolumeAttachments }} + - --reserved-volume-attachments={{ . }} + {{- end }} + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + {{- range .Values.node.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: {{ .Values.node.kubeletPath }} + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: cryptsetup + mountPath: /run/cryptsetup + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.node.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + {{- range .Values.sidecars.nodeDriverRegistrar.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ printf "%s/plugins/ebs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: probe-dir + mountPath: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.additionalArgs }} + - {{ . }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: cryptsetup + hostPath: + path: /run/cryptsetup + type: Directory + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.node.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml index 816fdf66e..bff6577b3 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml @@ -21,3 +21,6 @@ rules: - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments/status" ] verbs: [ "patch" ] + {{- with .Values.sidecars.attacher.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml index 3ca368efb..2b7295aaf 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml @@ -8,4 +8,10 @@ metadata: rules: - apiGroups: [""] resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get"] diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml index 0fb7ded0f..b67c65844 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml @@ -30,9 +30,12 @@ rules: - apiGroups: [ "" ] resources: [ "nodes" ] verbs: [ "get", "list", "watch" ] - - apiGroups: [ "coordination.k8s.io" ] - resources: [ "leases" ] - verbs: [ "get", "watch", "list", "delete", "update", "create" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattributesclasses" ] + verbs: [ "get" ] + {{- with .Values.sidecars.provisioner.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml index 065f3aba2..81858af34 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml @@ -29,3 +29,9 @@ rules: - apiGroups: [ "" ] resources: [ "pods" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattributesclasses" ] + verbs: [ "get", "list", "watch" ] + {{- with .Values.sidecars.resizer.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml index 38e688a8a..697e818d9 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml @@ -24,4 +24,7 @@ rules: verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotcontents/status" ] - verbs: [ "update" ] + verbs: [ "update", "patch" ] + {{- with .Values.sidecars.snapshotter.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml index 0d79331ac..89468b1ca 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml @@ -6,8 +6,15 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.controller.replicaCount }} + {{- if or (kindIs "float64" .Values.controller.revisionHistoryLimit) (kindIs "int64" .Values.controller.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + {{- end }} {{- with .Values.controller.updateStrategy }} strategy: {{- toYaml . | nindent 4 }} @@ -26,7 +33,7 @@ spec: {{- end }} {{- if .Values.controller.podAnnotations }} annotations: - {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- tpl ( .Values.controller.podAnnotations | toYaml ) . | nindent 8 }} {{- end }} spec: nodeSelector: @@ -75,7 +82,7 @@ spec: {{- if .Values.controller.extraVolumeTags }} {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} {{- end }} - {{- with .Values.controller.k8sTagClusterId }} + {{- with (tpl (default "" .Values.controller.k8sTagClusterId) . ) }} - --k8s-tag-cluster-id={{ . }} {{- end }} {{- if and (.Values.controller.enableMetrics) (not .Values.controller.httpEndpoint) }} @@ -87,9 +94,18 @@ spec: {{- if .Values.controller.sdkDebugLog }} - --aws-sdk-debug-log=true {{- end}} + {{- if .Values.controller.batching }} + - --batching=true + {{- end}} {{- with .Values.controller.loggingFormat }} - --logging-format={{ . }} {{- end }} + {{- with .Values.controller.userAgentExtra }} + - --user-agent-extra={{ . }} + {{- end }} + {{- if .Values.controller.otelTracing }} + - --enable-otel-tracing=true + {{- end}} - --v={{ .Values.controller.logLevel }} {{- range .Values.controller.additionalArgs }} - {{ . }} @@ -101,18 +117,20 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + {{- with .Values.awsAccessSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: aws-secret - key: key_id + name: {{ .name }} + key: {{ .keyId }} optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: aws-secret - key: access_key + name: {{ .name }} + key: {{ .accessKey }} optional: true + {{- end }} - name: AWS_EC2_ENDPOINT valueFrom: configMapKeyRef: @@ -129,10 +147,16 @@ spec: {{- with .Values.controller.env }} {{- . | toYaml | nindent 12 }} {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} + {{- with .Values.controller.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -176,6 +200,9 @@ spec: image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.provisioner.image.repository .Values.sidecars.provisioner.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.provisioner.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.provisioner.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.provisioner.logLevel }} - --feature-gates=Topology=true @@ -195,6 +222,14 @@ spec: {{- end }} {{- end }} - --default-fstype={{ .Values.controller.defaultFsType }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.provisioner.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.provisioner.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -204,10 +239,10 @@ spec: {{- with .Values.sidecars.provisioner.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -223,6 +258,9 @@ spec: image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.attacher.image.repository .Values.sidecars.attacher.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.attacher.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.attacher.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.attacher.logLevel }} - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }} @@ -237,6 +275,14 @@ spec: - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }} {{- end }} {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.attacher.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.attacher.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -246,10 +292,10 @@ spec: {{- with .Values.sidecars.attacher.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -271,6 +317,14 @@ spec: {{- if .Values.controller.extraCreateMetadata }} - --extra-create-metadata {{- end}} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.snapshotter.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.snapshotter.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -280,10 +334,10 @@ spec: {{- with .Values.sidecars.snapshotter.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -296,13 +350,94 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} + {{- if (.Values.controller.volumeModificationFeature).enabled }} + - name: volumemodifier + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.volumemodifier.image.repository .Values.sidecars.volumemodifier.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.volumemodifier.image.pullPolicy }} + args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.volumemodifier.additionalArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.volumemodifier.logLevel }} + - --leader-election={{ .Values.sidecars.volumemodifier.leaderElection.enabled | required "leader election state for csi-volumemodifier is required, must be set to true || false." }} + {{- if .Values.sidecars.volumemodifier.leaderElection.enabled }} + {{- if .Values.sidecars.volumemodifier.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.volumemodifier.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.volumemodifier.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.volumemodifier.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.volumemodifier.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.volumemodifier.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + {{- range .Values.sidecars.volumemodifier.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.volumemodifier.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.volumemodifier.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.volumemodifier.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} - name: csi-resizer image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.resizer.image.repository .Values.sidecars.resizer.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.resizer.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.resizer.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.resizer.logLevel }} - --handle-volume-inuse-error=false + {{- with .Values.sidecars.resizer.leaderElection }} + - --leader-election={{ .enabled | default true }} + {{- if .leaseDuration }} + - --leader-election-lease-duration={{ .leaseDuration }} + {{- end }} + {{- if .renewDeadline }} + - --leader-election-renew-deadline={{ .renewDeadline }} + {{- end }} + {{- if .retryPeriod }} + - --leader-election-retry-period={{ .retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-workers)" (join " " .Values.sidecars.resizer.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + {{- end }} + {{- range .Values.sidecars.resizer.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -312,10 +447,10 @@ spec: {{- with .Values.sidecars.resizer.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -332,10 +467,13 @@ spec: imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} + {{- range .Values.sidecars.livenessProbe.additionalArgs }} + - {{ . }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -355,7 +493,15 @@ spec: {{- end }} volumes: - name: socket-dir + {{- if .Values.controller.socketDirVolume }} + {{- toYaml .Values.controller.socketDirVolume | nindent 10 }} + {{- else }} emptyDir: {} + {{- end }} {{- with .Values.controller.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.controller.dnsConfig }} + dnsConfig: + {{- toYaml .Values.controller.dnsConfig | nindent 4 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml new file mode 100644 index 000000000..a58595726 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml @@ -0,0 +1,11 @@ +{{- if .Values.defaultStorageClass.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ebs-csi-default-sc + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml index 1dcdf4ddc..d68bd7ab9 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml @@ -37,6 +37,6 @@ spec: endpoints: - targetPort: 3301 path: /metrics - interval: 15s + interval: {{ .Values.controller.serviceMonitor.interval | default "15s"}} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml index 921b51cfb..9a2c2c81a 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml @@ -1,184 +1,13 @@ -{{- if .Values.node.enableWindows }} -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node-windows - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{ toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: windows - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - --v={{ .Values.node.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: C:\var\lib\kubelet - mountPropagation: "None" - - name: plugin-dir - mountPath: C:\csi - - name: csi-proxy-disk-pipe - mountPath: \\.\pipe\csi-proxy-disk-v1 - - name: csi-proxy-volume-pipe - mountPath: \\.\pipe\csi-proxy-volume-v1 - - name: csi-proxy-filesystem-pipe - mountPath: \\.\pipe\csi-proxy-filesystem-v1 - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: unix:/csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud\csi.sock - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar.exe - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - - name: registration-dir - mountPath: C:\registration - - name: probe-dir - mountPath: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=unix:/csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: C:\var\lib\kubelet - type: Directory - - name: plugin-dir - hostPath: - path: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: C:\var\lib\kubelet\plugins_registry - type: Directory - - name: csi-proxy-disk-pipe - hostPath: - path: \\.\pipe\csi-proxy-disk-v1 - type: "" - - name: csi-proxy-volume-pipe - hostPath: - path: \\.\pipe\csi-proxy-volume-v1 - type: "" - - name: csi-proxy-filesystem-pipe - hostPath: - path: \\.\pipe\csi-proxy-filesystem-v1 - type: "" - - name: probe-dir - emptyDir: {} +{{$defaultArgs := dict + "NodeName" "ebs-csi-node" +}} +{{- include "node-windows" (deepCopy $ | mustMerge $defaultArgs) -}} +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node-windows" (deepCopy $ | mustMerge $args) -}} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml index ecc6412d8..a891513b6 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml @@ -1,206 +1,46 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{- toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - {{- with .Values.node.securityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - "--kms-addr={{ .Values.kms.keyServiceName }}.{{ .Values.kms.keyServiceNamespace | default .Release.Namespace }}:{{ .Values.kms.keyServicePort }}" - - --v={{ .Values.node.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: {{ .Values.node.kubeletPath }} - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: device-dir - mountPath: /dev - - name: cryptsetup - mountPath: /run/cryptsetup - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.node.containerSecurityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: {{ printf "%s/plugins/aws.csi.confidential.cloud/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: probe-dir - mountPath: {{ printf "%s/plugins/aws.csi.confidential.cloud/" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=/csi/csi.sock - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.livenessProbe.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: {{ .Values.node.kubeletPath }} - type: Directory - - name: plugin-dir - hostPath: - path: {{ printf "%s/plugins/aws.csi.confidential.cloud/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory - - name: probe-dir - emptyDir: {} - - name: cryptsetup - hostPath: - path: /run/cryptsetup - type: Directory +{{$defaultArgs := dict + "NodeName" "ebs-csi-node" +}} +{{- include "node" (deepCopy $ | mustMerge $defaultArgs) -}} +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} +{{- if .Values.a1CompatibilityDaemonSet }} +{{$args := dict + "NodeName" "ebs-csi-node-a1compat" + "Values" (dict + "image" (dict + "tag" (printf "%s-a1compat" (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString))) + ) + "node" (dict + "affinity" (dict + "nodeAffinity" (dict + "requiredDuringSchedulingIgnoredDuringExecution" (dict + "nodeSelectorTerms" (list + (dict "matchExpressions" (list + (dict + "key" "eks.amazonaws.com/compute-type" + "operator" "NotIn" + "values" (list "fargate") + ) + (dict + "key" "node.kubernetes.io/instance-type" + "operator" "In" + "values" (list "a1.medium" "a1.large" "a1.xlarge" "a1.2xlarge" "a1.4xlarge") + ) + )) + ) + ) + ) + ) + ) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml index 6f73fa222..0a1e97cc0 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -1,8 +1,4 @@ -{{- if .Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} apiVersion: policy/v1 -{{- else }} -apiVersion: policy/v1beta1 -{{- end }} kind: PodDisruptionBudget metadata: name: ebs-csi-controller diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml new file mode 100644 index 000000000..1ec62bb49 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml @@ -0,0 +1,11 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ .Release.Namespace }} + name: ebs-csi-leases-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml new file mode 100644 index 000000000..88fded8a3 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml @@ -0,0 +1,15 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-leases-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: ebs-csi-leases-role + apiGroup: rbac.authorization.k8s.io diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml index a5b1102b4..d819f5493 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -15,4 +15,7 @@ metadata: #annotations: # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role {{- end }} +{{- if .Values.controller.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.controller.serviceAccount.automountServiceAccountToken }} +{{- end }} {{- end -}} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml index fb85abedf..9f3c7c7e1 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml @@ -10,4 +10,7 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- if .Values.node.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.node.serviceAccount.automountServiceAccountToken }} +{{- end }} {{- end -}} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml index 0db3046aa..59551898e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml @@ -8,6 +8,9 @@ metadata: {{- with .annotations }} annotations: {{- . | toYaml | trim | nindent 4 }} {{- end }} + {{- with .labels }} + labels: {{- . | toYaml | trim | nindent 4 }} + {{- end }} driver: aws.csi.confidential.cloud deletionPolicy: {{ .deletionPolicy }} {{- with .parameters }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml index defdd4d83..0c4a68857 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml @@ -5,7 +5,7 @@ image: repository: ghcr.io/edgelesssys/constellation/aws-csi-driver # Overrides the image tag whose default is v{{ .Chart.AppVersion }} - tag: "v1.1.0@sha256:3e2c394f2397455516948efcc9a4b87cbaeabf14df31702d5905ce08a53995f2" + tag: "v1.2.0@sha256:9477e8ed37989c46963d57d24de5ddbc7ab965b1685d709d06c1ae536b23b5b1" pullPolicy: Always # -- Custom labels to add into metadata @@ -24,8 +24,12 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: "v3.4.1-eks-1-26-7@sha256:adfcb04433d1824f62dde0365877d0f7b7a2eaebc45670cbab7e0c1f07ba0607" + tag: "v4.0.1-eks-1-30-4@sha256:0cf0d02211632c6b947f110e9f3f13f782eea1cfb7b990191d78ad032b2c2d77" logLevel: 2 + # Additional parameters provided by external-provisioner. + additionalArgs: [] + # Grant additional permissions to external-provisioner + additionalClusterRoleRules: resources: {} # Tune leader lease election for csi-provisioner. # Leader election is on by default. @@ -45,7 +49,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher - tag: "v4.2.0-eks-1-26-7@sha256:4b0d6e8758a0213ec942381b9577d2b3e971b545dc9e3fb59973f7992763d85f" + tag: "v4.5.1-eks-1-30-4@sha256:d68034351f65101d2a8506a5c583c5c923238aa93ba9719e779c0eb6a4b33993" # Tune leader lease election for csi-attacher. # Leader election is on by default. leaderElection: @@ -57,6 +61,10 @@ sidecars: # renewDeadline: "10s" # retryPeriod: "5s" logLevel: 2 + # Additional parameters provided by external-attacher. + additionalArgs: [] + # Grant additional permissions to external-attacher + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -68,8 +76,12 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter - tag: "v6.2.1-eks-1-26-7@sha256:b8071f45885f1838387edb04a1d164680dcec8d656de682624ddc59d30ba660b" + tag: "v7.0.2-eks-1-30-4@sha256:9a33488c2cd691d4df454fbc0118e532cbd8aacf99856bdf395507fdae2421dc" logLevel: 2 + # Additional parameters provided by csi-snapshotter. + additionalArgs: [] + # Grant additional permissions to csi-snapshotter + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -78,7 +90,9 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: "v2.9.0-eks-1-26-7@sha256:d9e11b42ae5f4f2f7ea9034e68040997cdbb04ae9e188aa897f76ae92698d78a" + tag: "v2.12.0-eks-1-30-4@sha256:665d64a8e1124ecd95e08626ddd140154be30a95c6574d423d66cf262d28cc9c" + # Additional parameters provided by livenessprobe. + additionalArgs: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -88,8 +102,22 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer - tag: "v1.7.0-eks-1-26-7@sha256:81672f19d1da5cdff8d2068d8d69776067a1e5c31537ab3282d95dff34d581b6" + tag: "v1.10.1-eks-1-30-4@sha256:2aef6bf851fc3fa8e03c7a3efc9d3adb2ae1cb1746f88fb8a7559f8ca44bf188" + # Tune leader lease election for csi-resizer. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" logLevel: 2 + # Additional parameters provided by external-resizer. + additionalArgs: [] + # Grant additional permissions to external-resizer + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -99,8 +127,40 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: "v2.7.0-eks-1-26-7@sha256:6ad0cae2ae91453f283a44e9b430e475b8a9fa3d606aec9a8b09596fffbcd2c9" + tag: "v2.10.1-eks-1-30-4@sha256:518ed9cba6258735a25d2b896dc65d34a41e22f6785550a7e24e2f2dbd6a48b5" logLevel: 2 + # Additional parameters provided by node-driver-registrar. + additionalArgs: [] + resources: {} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + volumemodifier: + env: [] + image: + pullPolicy: IfNotPresent + repository: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s + tag: "v0.3.0@sha256:c9e4e64e721b8fea4ba34502ac9f8fb83bd1571117276f553ac4595e0c5a3bf8" + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Additional parameters provided by volume-modifier-for-k8s. + additionalArgs: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -114,7 +174,16 @@ imagePullSecrets: [] nameOverride: fullnameOverride: +awsAccessSecret: + name: aws-secret + keyId: key_id + accessKey: access_key + controller: + batching: true + volumeModificationFeature: + enabled: false + # Additional parameters provided by aws-ebs-csi-driver controller. additionalArgs: [] sdkDebugLog: false loggingFormat: text @@ -165,12 +234,15 @@ controller: # Additional labels for ServiceMonitor object labels: release: prometheus + interval: "15s" # If set to true, AWS API call metrics will be exported to the following # TCP endpoint: "0.0.0.0:3301" # --- # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). k8sTagClusterId: logLevel: 2 + userAgentExtra: "helm" + deploymentAnnotations: {} nodeSelector: node-role.kubernetes.io/control-plane: "" podAnnotations: {} @@ -182,6 +254,9 @@ controller: # region: us-east-1 region: replicaCount: 2 + revisionHistoryLimit: 10 + socketDirVolume: + emptyDir: {} updateStrategy: type: RollingUpdate rollingUpdate: @@ -195,13 +270,15 @@ controller: cpu: 10m memory: 40Mi limits: - cpu: 100m memory: 256Mi serviceAccount: # A service account will be created for you if set to true. Set to false if you want to use your own. create: true name: ebs-csi-controller-sa annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role + automountServiceAccountToken: true tolerations: - key: CriticalAddonsOnly operator: Exists @@ -235,8 +312,18 @@ controller: runAsUser: 1000 runAsGroup: 1000 fsGroup: 1000 + # Add additional volume mounts on the controller with controller.volumes and controller.volumeMounts volumes: [] + # Add additional volumes to be mounted onto the controller: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # --- # securityContext on the controller container (see sidecars for securityContext on sidecar containers) containerSecurityContext: readOnlyRootFilesystem: true @@ -249,6 +336,13 @@ controller: # - name: wait # image: busybox # command: [ 'sh', '-c', "sleep 20" ] + # Enable opentelemetry tracing for the plugin running on the daemonset + otelTracing: {} + # otelServiceName: ebs-csi-controller + # otelExporterEndpoint: "http://localhost:4317" + + # Enable dnsConfig for the controller and node pods + dnsConfig: {} node: env: [] @@ -257,16 +351,26 @@ node: loggingFormat: text logLevel: 2 priorityClassName: + additionalArgs: [] affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: node.kubernetes.io/instance-type + operator: NotIn + values: + - a1.medium + - a1.large + - a1.xlarge + - a1.2xlarge + - a1.4xlarge nodeSelector: {} + daemonSetAnnotations: {} podAnnotations: {} podLabels: {} tolerateAllTaints: true @@ -279,19 +383,32 @@ node: cpu: 10m memory: 40Mi limits: - cpu: 100m memory: 256Mi + revisionHistoryLimit: 10 + probeDirVolume: + emptyDir: {} serviceAccount: create: true name: ebs-csi-node-sa annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role + automountServiceAccountToken: true + # Enable the linux daemonset creation + enableLinux: true enableWindows: false + # The number of attachment slots to reserve for system use (and not to be used for CSI volumes) + # When this parameter is not specified (or set to -1), the EBS CSI Driver will attempt to determine the number of reserved slots via heuristic + # Cannot be specified at the same time as `node.volumeAttachLimit` + reservedVolumeAttachments: # The "maximum number of attachable volumes" per node + # Cannot be specified at the same time as `node.reservedVolumeAttachments` volumeAttachLimit: updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: "10%" + hostNetwork: false # securityContext on the node pod securityContext: # The node pod must be run as root to bind to the registration/driver sockets @@ -299,10 +416,38 @@ node: runAsUser: 0 runAsGroup: 0 fsGroup: 0 + # Add additional volume mounts on the node pods with node.volumes and node.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the node pods: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # --- # securityContext on the node container (see sidecars for securityContext on sidecar containers) containerSecurityContext: readOnlyRootFilesystem: true privileged: true + # Enable opentelemetry tracing for the plugin running on the daemonset + otelTracing: {} + # otelServiceName: ebs-csi-node + # otelExporterEndpoint: "http://localhost:4317" + +additionalDaemonSets: + # Additional node DaemonSets, using the node config structure + # See docs/additional-daemonsets.md for more information + # + # example: + # nodeSelector: + # node.kubernetes.io/instance-type: c5.large + # volumeAttachLimit: 15 + +# Enable compatibility for the A1 instance family via use of an AL2-based image in a separate DaemonSet +# a1CompatibilityDaemonSet: true # Create Constellation default StorageClasses createStorageClass: true @@ -323,12 +468,18 @@ storageClasses: [] # parameters: # encrypted: "true" +defaultStorageClass: + enabled: false + volumeSnapshotClasses: [] # Add VolumeSnapshotClass resources like: # - name: ebs-vsc # # annotation metadata # annotations: # snapshot.storage.kubernetes.io/is-default-class: "true" +# # label metadata +# labels: +# my-label-is: supercool # # deletionPolicy must be specified # deletionPolicy: Delete # parameters: @@ -337,3 +488,8 @@ volumeSnapshotClasses: [] # Intended for use with older clusters that cannot easily replace the CSIDriver object # This parameter should always be false for new installations useOldCSIDriver: false + +helmTester: + enabled: true + # Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml + image: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240311-b09cdeb92c-master" diff --git a/internal/constellation/helm/update-csi-charts.sh b/internal/constellation/helm/update-csi-charts.sh index 96dba4a02..16debb98d 100755 --- a/internal/constellation/helm/update-csi-charts.sh +++ b/internal/constellation/helm/update-csi-charts.sh @@ -68,7 +68,7 @@ download_chart() { } ## AWS CSI Driver -download_chart "https://github.com/edgelesssys/constellation-aws-ebs-csi-driver" "v1.1.1" "charts/aws-ebs-csi-driver" "aws-csi-driver" +download_chart "https://github.com/edgelesssys/constellation-aws-ebs-csi-driver" "v1.2.0" "charts/aws-ebs-csi-driver" "aws-csi-driver" ## Azure CSI Driver download_chart "https://github.com/edgelesssys/constellation-azuredisk-csi-driver" "v1.3.0" "charts/edgeless" "azuredisk-csi-driver"