From d8de0a61316932eaee4cb80a65401bd25093b260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=9C=E6=9D=BE?= Date: Fri, 10 May 2024 11:09:32 +0800 Subject: [PATCH 1/3] feat: supplement helm deploy for edge --- deploy/chart/templates/plugin.yaml | 75 ++++++++++++++++-------------- deploy/chart/values-edge.yaml | 33 +++++++++++-- deploy/chart/values.yaml | 10 ++++ 3 files changed, 77 insertions(+), 41 deletions(-) diff --git a/deploy/chart/templates/plugin.yaml b/deploy/chart/templates/plugin.yaml index 34d4130c8..2bbd77497 100644 --- a/deploy/chart/templates/plugin.yaml +++ b/deploy/chart/templates/plugin.yaml @@ -1,17 +1,23 @@ -{{- if .Values.plugin.enabled -}} +{{- range $key, $nodepool := .Values.nodepools -}} +{{- if and $.Values.plugin.enabled (ne $nodepool nil) }} +--- +{{- $values := deepCopy $.Values }} +{{- $nodepool := mergeOverwrite $values $nodepool }} kind: DaemonSet apiVersion: apps/v1 metadata: - name: csi-plugin - namespace: {{ .Release.Namespace }} + name: {{ $nodepool.pluginname }} + namespace: {{ $.Release.Namespace }} spec: selector: matchLabels: app: csi-plugin + nodepool: {{ $key }} template: metadata: labels: app: csi-plugin + nodepool: {{ $key }} spec: tolerations: - operator: Exists @@ -19,11 +25,7 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet +{{- toYaml $nodepool.nodeSelectorTerms | nindent 12 }} nodeSelector: kubernetes.io/os: linux serviceAccountName: alicloud-csi-node @@ -34,9 +36,9 @@ spec: dnsPolicy: ClusterFirst initContainers: - name: init - image: {{ include "workerImageSpec" (list .Values.images "pluginInit") }} + image: {{ include "workerImageSpec" (list $nodepool.images "pluginInit") }} args: - - "--driver={{ include "enabledPlugins" .Values.csi }}" + - "--driver={{ include "enabledPlugins" $nodepool.csi }}" securityContext: privileged: true allowPrivilegeEscalation: true @@ -57,10 +59,10 @@ spec: - name: ossconnectordir mountPath: /host/usr/ containers: -{{- range $key, $val := .Values.csi }} +{{- range $key, $val := $nodepool.csi }} {{- if $val.enabled }} - name: {{$key}}-driver-registrar - image: {{ include "workerImageSpec" (list $.Values.images "pluginRegistrar") }} + image: {{ include "workerImageSpec" (list $nodepool.images "pluginRegistrar") }} resources: requests: cpu: 10m @@ -83,15 +85,15 @@ spec: securityContext: privileged: true allowPrivilegeEscalation: true - image: {{ include "workerImageSpec" (list .Values.images "plugin") }} + image: {{ include "workerImageSpec" (list $nodepool.images "plugin") }} args: - "--endpoint=unix://csi/driverplugin.csi.alibabacloud.com-replace/csi.sock" - "--v=2" - - "--driver={{ include "enabledPlugins" .Values.csi }}" -{{- if .Values.deploy.featureGates }} - - "--feature-gates={{ .Values.deploy.featureGates }}" + - "--driver={{ include "enabledPlugins" $nodepool.csi }}" +{{- if $nodepool.deploy.featureGates }} + - "--feature-gates={{ $nodepool.deploy.featureGates }}" {{- end -}} -{{- if not .Values.deploy.ecs }} +{{- if not $nodepool.deploy.ecs }} - --nodeid=$(KUBE_NODE_NAME) {{- end }} env: @@ -103,18 +105,18 @@ spec: - name: SERVICE_TYPE value: "plugin" - name: "DEFAULT_REGISTRY" - value: {{ .Values.images.workerRegistry | default .Values.images.registry | quote }} -{{- if .Values.deploy.ecsEndpoint }} + value: {{ $nodepool.images.workerRegistry | default $nodepool.images.registry | quote }} +{{- if $nodepool.deploy.ecsEndpoint }} - name: ECS_ENDPOINT - value: {{ .Values.deploy.ecsEndpoint | quote }} + value: {{ $nodepool.deploy.ecsEndpoint | quote }} {{- end -}} -{{- if .Values.deploy.privateCloud }} +{{- if $nodepool.deploy.privateCloud }} - name: ALICLOUD_CLIENT_SCHEME value: HTTP - name: PRIVATE_CLOUD_TAG value: "true" {{- end -}} -{{- include "akEnv" .Values.deploy.accessKey | nindent 12 }} +{{- include "akEnv" $nodepool.deploy.accessKey | nindent 12 }} resources: requests: cpu: 100m @@ -165,16 +167,16 @@ spec: mountPropagation: "HostToContainer" - mountPath: /run/kata-containers/shared/direct-volumes name: kata-direct-volumes -{{- if .Values.deploy.ack }} +{{- if $nodepool.deploy.ack }} - mountPath: /var/addon name: addon-token readOnly: true {{- end -}} -{{- if .Values.csi.oss.enabled }} +{{- if $nodepool.csi.oss.enabled }} - mountPath: /host/var/run/ossfs name: ossfs-metrics-dir {{- end -}} -{{- if .Values.csi.nas.enabled }} +{{- if $nodepool.csi.nas.enabled }} - mountPath: /host/var/run/efc name: efc-metrics-dir {{- end }} @@ -187,12 +189,12 @@ spec: - name: host-mnt mountPath: /mnt mountPropagation: "Bidirectional" -{{- if .Values.csi.local.enabled }} +{{- if $nodepool.csi.local.enabled }} - name: local-csi-plugin securityContext: privileged: true allowPrivilegeEscalation: true - image: {{ include "workerImageSpec" (list .Values.images "localPlugin") }} + image: {{ include "workerImageSpec" (list $nodepool.images "localPlugin") }} args: - --endpoint=unix://csi/csi.sock - --http-endpoint=:11261 @@ -206,15 +208,15 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName -{{- if .Values.deploy.ecsEndpoint }} +{{- if $nodepool.deploy.ecsEndpoint }} - name: ECS_ENDPOINT - value: {{ .Values.deploy.ecsEndpoint | quote }} + value: {{ $nodepool.deploy.ecsEndpoint | quote }} {{- end -}} -{{- if .Values.deploy.privateCloud }} +{{- if $nodepool.deploy.privateCloud }} - name: ALICLOUD_CLIENT_SCHEME value: HTTP {{- end -}} -{{- include "akEnv" .Values.deploy.accessKey | nindent 12 }} +{{- include "akEnv" $nodepool.deploy.accessKey | nindent 12 }} resources: requests: cpu: 100m @@ -250,7 +252,7 @@ spec: - name: host-dev mountPath: /dev mountPropagation: "HostToContainer" -{{- if .Values.deploy.ack }} +{{- if $nodepool.deploy.ack }} - mountPath: /var/addon name: addon-token readOnly: true @@ -265,13 +267,13 @@ spec: mountPropagation: "Bidirectional" {{- end }} volumes: -{{- if .Values.csi.oss.enabled }} +{{- if $nodepool.csi.oss.enabled }} - name: ossfs-metrics-dir hostPath: path: /var/run/ossfs type: DirectoryOrCreate {{- end -}} -{{- if .Values.csi.nas.enabled }} +{{- if $nodepool.csi.nas.enabled }} - name: efc-metrics-dir hostPath: path: /var/run/efc @@ -281,7 +283,7 @@ spec: hostPath: path: /run/csi-tool type: DirectoryOrCreate -{{- if .Values.csi.local.enabled }} +{{- if $nodepool.csi.local.enabled }} - name: tls-token-dir secret: defaultMode: 420 @@ -335,7 +337,7 @@ spec: path: /run/kata-containers/shared/direct-volumes type: DirectoryOrCreate name: kata-direct-volumes -{{- if .Values.deploy.ack }} +{{- if $nodepool.deploy.ack }} - name: addon-token secret: defaultMode: 420 @@ -350,3 +352,4 @@ spec: maxUnavailable: 20% type: RollingUpdate {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/deploy/chart/values-edge.yaml b/deploy/chart/values-edge.yaml index b01506832..61caa5c16 100644 --- a/deploy/chart/values-edge.yaml +++ b/deploy/chart/values-edge.yaml @@ -1,11 +1,34 @@ -deploy: - ecs: false - +# deployment csi: - disk: - enabled: false local: enabled: true oss: controller: enabled: false + +# daemonset +nodepools: + default: + pluginname: csi-plugin-cloud + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: alibabacloud.com/is-edge-worker + operator: In + values: + - "false" + edge: + pluginname: csi-plugin + csi: + disk: + enabled: false + nodeSelectorTerms: + - matchExpressions: + - key: alibabacloud.com/is-edge-worker + operator: In + values: + - "true" + # other nodepools \ No newline at end of file diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 8781f1292..45347b941 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -23,6 +23,16 @@ csi: controller: enabled: true +nodepools: + default: + pluginname: csi-plugin + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + defaultStorageClass: enabled: true From b0e5f17278dc0ffd15d8e4ffc77bde172a077ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=9C=E6=9D=BE?= Date: Tue, 21 May 2024 13:35:01 +0800 Subject: [PATCH 2/3] chore: rename nodepool to nodePool --- deploy/chart/templates/plugin.yaml | 66 +++++++++++++++--------------- deploy/chart/values-edge.yaml | 6 ++- deploy/chart/values.yaml | 2 +- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/deploy/chart/templates/plugin.yaml b/deploy/chart/templates/plugin.yaml index 2bbd77497..2ee66ad70 100644 --- a/deploy/chart/templates/plugin.yaml +++ b/deploy/chart/templates/plugin.yaml @@ -1,12 +1,12 @@ -{{- range $key, $nodepool := .Values.nodepools -}} -{{- if and $.Values.plugin.enabled (ne $nodepool nil) }} +{{- range $key, $nodePool := .Values.nodePools -}} +{{- if and $.Values.plugin.enabled (ne $nodePool nil) }} --- {{- $values := deepCopy $.Values }} -{{- $nodepool := mergeOverwrite $values $nodepool }} +{{- $nodePool := mergeOverwrite $values $nodePool }} kind: DaemonSet apiVersion: apps/v1 metadata: - name: {{ $nodepool.pluginname }} + name: {{ $nodePool.pluginname }} namespace: {{ $.Release.Namespace }} spec: selector: @@ -25,7 +25,7 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: -{{- toYaml $nodepool.nodeSelectorTerms | nindent 12 }} +{{- toYaml $nodePool.nodeSelectorTerms | nindent 12 }} nodeSelector: kubernetes.io/os: linux serviceAccountName: alicloud-csi-node @@ -36,9 +36,9 @@ spec: dnsPolicy: ClusterFirst initContainers: - name: init - image: {{ include "workerImageSpec" (list $nodepool.images "pluginInit") }} + image: {{ include "workerImageSpec" (list $nodePool.images "pluginInit") }} args: - - "--driver={{ include "enabledPlugins" $nodepool.csi }}" + - "--driver={{ include "enabledPlugins" $nodePool.csi }}" securityContext: privileged: true allowPrivilegeEscalation: true @@ -59,10 +59,10 @@ spec: - name: ossconnectordir mountPath: /host/usr/ containers: -{{- range $key, $val := $nodepool.csi }} +{{- range $key, $val := $nodePool.csi }} {{- if $val.enabled }} - name: {{$key}}-driver-registrar - image: {{ include "workerImageSpec" (list $nodepool.images "pluginRegistrar") }} + image: {{ include "workerImageSpec" (list $nodePool.images "pluginRegistrar") }} resources: requests: cpu: 10m @@ -85,15 +85,15 @@ spec: securityContext: privileged: true allowPrivilegeEscalation: true - image: {{ include "workerImageSpec" (list $nodepool.images "plugin") }} + image: {{ include "workerImageSpec" (list $nodePool.images "plugin") }} args: - "--endpoint=unix://csi/driverplugin.csi.alibabacloud.com-replace/csi.sock" - "--v=2" - - "--driver={{ include "enabledPlugins" $nodepool.csi }}" -{{- if $nodepool.deploy.featureGates }} - - "--feature-gates={{ $nodepool.deploy.featureGates }}" + - "--driver={{ include "enabledPlugins" $nodePool.csi }}" +{{- if $nodePool.deploy.featureGates }} + - "--feature-gates={{ $nodePool.deploy.featureGates }}" {{- end -}} -{{- if not $nodepool.deploy.ecs }} +{{- if not $nodePool.deploy.ecs }} - --nodeid=$(KUBE_NODE_NAME) {{- end }} env: @@ -105,18 +105,18 @@ spec: - name: SERVICE_TYPE value: "plugin" - name: "DEFAULT_REGISTRY" - value: {{ $nodepool.images.workerRegistry | default $nodepool.images.registry | quote }} -{{- if $nodepool.deploy.ecsEndpoint }} + value: {{ $nodePool.images.workerRegistry | default $nodePool.images.registry | quote }} +{{- if $nodePool.deploy.ecsEndpoint }} - name: ECS_ENDPOINT - value: {{ $nodepool.deploy.ecsEndpoint | quote }} + value: {{ $nodePool.deploy.ecsEndpoint | quote }} {{- end -}} -{{- if $nodepool.deploy.privateCloud }} +{{- if $nodePool.deploy.privateCloud }} - name: ALICLOUD_CLIENT_SCHEME value: HTTP - name: PRIVATE_CLOUD_TAG value: "true" {{- end -}} -{{- include "akEnv" $nodepool.deploy.accessKey | nindent 12 }} +{{- include "akEnv" $nodePool.deploy.accessKey | nindent 12 }} resources: requests: cpu: 100m @@ -167,16 +167,16 @@ spec: mountPropagation: "HostToContainer" - mountPath: /run/kata-containers/shared/direct-volumes name: kata-direct-volumes -{{- if $nodepool.deploy.ack }} +{{- if $nodePool.deploy.ack }} - mountPath: /var/addon name: addon-token readOnly: true {{- end -}} -{{- if $nodepool.csi.oss.enabled }} +{{- if $nodePool.csi.oss.enabled }} - mountPath: /host/var/run/ossfs name: ossfs-metrics-dir {{- end -}} -{{- if $nodepool.csi.nas.enabled }} +{{- if $nodePool.csi.nas.enabled }} - mountPath: /host/var/run/efc name: efc-metrics-dir {{- end }} @@ -189,12 +189,12 @@ spec: - name: host-mnt mountPath: /mnt mountPropagation: "Bidirectional" -{{- if $nodepool.csi.local.enabled }} +{{- if $nodePool.csi.local.enabled }} - name: local-csi-plugin securityContext: privileged: true allowPrivilegeEscalation: true - image: {{ include "workerImageSpec" (list $nodepool.images "localPlugin") }} + image: {{ include "workerImageSpec" (list $nodePool.images "localPlugin") }} args: - --endpoint=unix://csi/csi.sock - --http-endpoint=:11261 @@ -208,15 +208,15 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName -{{- if $nodepool.deploy.ecsEndpoint }} +{{- if $nodePool.deploy.ecsEndpoint }} - name: ECS_ENDPOINT - value: {{ $nodepool.deploy.ecsEndpoint | quote }} + value: {{ $nodePool.deploy.ecsEndpoint | quote }} {{- end -}} -{{- if $nodepool.deploy.privateCloud }} +{{- if $nodePool.deploy.privateCloud }} - name: ALICLOUD_CLIENT_SCHEME value: HTTP {{- end -}} -{{- include "akEnv" $nodepool.deploy.accessKey | nindent 12 }} +{{- include "akEnv" $nodePool.deploy.accessKey | nindent 12 }} resources: requests: cpu: 100m @@ -252,7 +252,7 @@ spec: - name: host-dev mountPath: /dev mountPropagation: "HostToContainer" -{{- if $nodepool.deploy.ack }} +{{- if $nodePool.deploy.ack }} - mountPath: /var/addon name: addon-token readOnly: true @@ -267,13 +267,13 @@ spec: mountPropagation: "Bidirectional" {{- end }} volumes: -{{- if $nodepool.csi.oss.enabled }} +{{- if $nodePool.csi.oss.enabled }} - name: ossfs-metrics-dir hostPath: path: /var/run/ossfs type: DirectoryOrCreate {{- end -}} -{{- if $nodepool.csi.nas.enabled }} +{{- if $nodePool.csi.nas.enabled }} - name: efc-metrics-dir hostPath: path: /var/run/efc @@ -283,7 +283,7 @@ spec: hostPath: path: /run/csi-tool type: DirectoryOrCreate -{{- if $nodepool.csi.local.enabled }} +{{- if $nodePool.csi.local.enabled }} - name: tls-token-dir secret: defaultMode: 420 @@ -337,7 +337,7 @@ spec: path: /run/kata-containers/shared/direct-volumes type: DirectoryOrCreate name: kata-direct-volumes -{{- if $nodepool.deploy.ack }} +{{- if $nodePool.deploy.ack }} - name: addon-token secret: defaultMode: 420 diff --git a/deploy/chart/values-edge.yaml b/deploy/chart/values-edge.yaml index 61caa5c16..095a670d0 100644 --- a/deploy/chart/values-edge.yaml +++ b/deploy/chart/values-edge.yaml @@ -7,7 +7,7 @@ csi: enabled: false # daemonset -nodepools: +nodePools: default: pluginname: csi-plugin-cloud nodeSelectorTerms: @@ -22,6 +22,8 @@ nodepools: - "false" edge: pluginname: csi-plugin + deploy: + ecs: false csi: disk: enabled: false @@ -31,4 +33,4 @@ nodepools: operator: In values: - "true" - # other nodepools \ No newline at end of file + # other nodePools \ No newline at end of file diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 45347b941..0f714fb1e 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -23,7 +23,7 @@ csi: controller: enabled: true -nodepools: +nodePools: default: pluginname: csi-plugin nodeSelectorTerms: From fdc352c59ede613e92fe2b2f671dd38f0c613823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=9C=E6=9D=BE?= Date: Thu, 23 May 2024 09:50:11 +0800 Subject: [PATCH 3/3] chore: rebase --- deploy/chart/templates/plugin.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/chart/templates/plugin.yaml b/deploy/chart/templates/plugin.yaml index 2ee66ad70..3092b4dd3 100644 --- a/deploy/chart/templates/plugin.yaml +++ b/deploy/chart/templates/plugin.yaml @@ -148,7 +148,7 @@ spec: mountPath: /var/lib/kubelet/ mountPropagation: "Bidirectional" {{- range $key := tuple "disk" "nas" "oss" }} - {{- with index $.Values.csi $key -}} + {{- with index $nodePool.csi $key -}} {{- if .enabled }} - name: {{ $key }}-plugin-dir mountPath: /csi/{{ $key }}plugin.csi.alibabacloud.com @@ -301,7 +301,7 @@ spec: hostPath: path: /var/lib/kubelet type: Directory -{{- range $key, $val := .Values.csi }} +{{- range $key, $val := $nodePool.csi }} {{- if $val.enabled }} - name: {{ $key }}-plugin-dir hostPath: