From b75a33da71988c7083537603609af33caaec7267 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:01:51 +0200 Subject: [PATCH 1/7] feat: define new values for SaaS credentials Allow benchmarks to configure and run against SaaS environment --- charts/zeebe-benchmark/values.yaml | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/charts/zeebe-benchmark/values.yaml b/charts/zeebe-benchmark/values.yaml index 14f4aa4..6f2ab00 100644 --- a/charts/zeebe-benchmark/values.yaml +++ b/charts/zeebe-benchmark/values.yaml @@ -40,6 +40,41 @@ global: auth: enabled: false +# Saas configuration to run benchmarks against Camunda SaaS environment +saas: + # Saas.enabled if true enables the benchmark to run against Camunda SaaS + enabled: false + + # Saas.credentials configuration to connect to a Camunda SaaS cluster + credentials: + # Saas.existingSecret can be used to configure the secret name that should be referenced by the benchmark + # applications to retrieve credential information. + # + # If this value is set, other credentials are not used. + # + # Credentials Secret need to follow the following format: + # + # apiVersion: v1 + # kind: Secret + # metadata: + # name: cloud-credentials + # type: Opaque + # stringData: + # clientId: hH55UFivfw-bbHAuPwN545oyv8tTdW0z + # clientSecret: xtHQB.zBLcQrw4GaP0k_ci~ePjbD8qVlYaFKNo__2a7ZJxL-DAVVHepq~X9elPRb + # zeebeAddress: e314a337-a462-4988-a3be-d1f2e153e034.zeebe.ultrawombat.com:443 + # authServer: https://login.cloud.ultrawombat.com/oauth/token + existingSecret: + + # Saas.credentials.clientId to define the clientId to connect + clientId: "" + # Saas.credentials.clientSecret to define the clientSecret to connect + clientSecret: "" + # Saas.credentials.zeebeAddress to define the address of the cluster (including port) + zeebeAddress: + # SaaS.credentials.authServer to define the authentication server to retrieve JWT tokens + authServer: "https://login.cloud.ultrawombat.com/oauth/token" + # Workers configuration for the to be deployed worker application # => New way to deploy workers <= workers: From c0949daf920f636def4c6edd8124db3e6d255c5e Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:02:48 +0200 Subject: [PATCH 2/7] feat: add helper to find correct secret name --- charts/zeebe-benchmark/templates/_helpers.tpl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/charts/zeebe-benchmark/templates/_helpers.tpl b/charts/zeebe-benchmark/templates/_helpers.tpl index a4b75d9..49a9c12 100644 --- a/charts/zeebe-benchmark/templates/_helpers.tpl +++ b/charts/zeebe-benchmark/templates/_helpers.tpl @@ -23,6 +23,19 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* +Create a default fully qualified credentials name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "zeebe-benchmark.credentials-name" -}} +{{- if .Values.saas.credentials.existingSecret }} +{{- .Values.saas.credentials.existingSecret }} +{{- else }} +{{- printf "%s-credentials" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + + {{/* Create chart name and version as used by the chart label. */}} From f9daab4340deefef7f9a989e603fd8cb48e4918c Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:04:08 +0200 Subject: [PATCH 3/7] feat: define credentials secret Secret that stores configured credential details for benchmark applications --- charts/zeebe-benchmark/templates/credentials.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 charts/zeebe-benchmark/templates/credentials.yaml diff --git a/charts/zeebe-benchmark/templates/credentials.yaml b/charts/zeebe-benchmark/templates/credentials.yaml new file mode 100644 index 0000000..91ccb3c --- /dev/null +++ b/charts/zeebe-benchmark/templates/credentials.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.saas.enabled (not .Values.saas.credentials.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "zeebe-benchmark.credentials-name" .}} +type: Opaque +stringData: + clientId: {{ .Values.saas.credentials.clientId }} + clientSecret: {{ .Values.saas.credentials.clientSecret }} + zeebeAddress: {{ .Values.saas.credentials.zeebeAddress }} + authServer: {{ .Values.saas.credentials.authServer }} +{{- end }} \ No newline at end of file From f454d88eabc40f7fc2ca044df1bfa3e68608ce42 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:11:15 +0200 Subject: [PATCH 4/7] test: add golden test for credentials --- ...golden-credentials-credentials.golden.yaml | 12 ++++++ .../test/saas_configuration_test.go | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 charts/zeebe-benchmark/test/golden/golden-credentials-credentials.golden.yaml create mode 100644 charts/zeebe-benchmark/test/saas_configuration_test.go diff --git a/charts/zeebe-benchmark/test/golden/golden-credentials-credentials.golden.yaml b/charts/zeebe-benchmark/test/golden/golden-credentials-credentials.golden.yaml new file mode 100644 index 0000000..8d5c129 --- /dev/null +++ b/charts/zeebe-benchmark/test/golden/golden-credentials-credentials.golden.yaml @@ -0,0 +1,12 @@ +--- +# Source: zeebe-benchmark/templates/credentials.yaml +apiVersion: v1 +kind: Secret +metadata: + name: benchmark-test-credentials +type: Opaque +stringData: + clientId: clientId + clientSecret: clientSecret + zeebeAddress: zeebeAddress + authServer: authServer \ No newline at end of file diff --git a/charts/zeebe-benchmark/test/saas_configuration_test.go b/charts/zeebe-benchmark/test/saas_configuration_test.go new file mode 100644 index 0000000..c4bd113 --- /dev/null +++ b/charts/zeebe-benchmark/test/saas_configuration_test.go @@ -0,0 +1,39 @@ +package test + +import ( + "benchmark-helm/charts/zeebe-benchmark/test/golden" + "path/filepath" + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/random" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func TestGoldenCredentials(t *testing.T) { + // Test which allows to verify also parent chart templates + // This makes sure that properties are correctly set + // OR configurations have been changed + + chartPath, err := filepath.Abs("../") + require.NoError(t, err) + templateNames := []string{"credentials"} + + for _, name := range templateNames { + suite.Run(t, &golden.TemplateGoldenTest{ + ChartPath: chartPath, + Release: "benchmark-test", + Namespace: "benchmark-" + strings.ToLower(random.UniqueId()), + GoldenFileName: "golden-credentials-" + name, + Templates: []string{"templates/" + name + ".yaml"}, + SetValues: map[string]string{ + "saas.enabled": "true", + "saas.credentials.clientId": "clientId", + "saas.credentials.clientSecret": "clientSecret", + "saas.credentials.authServer": "authServer", + "saas.credentials.zeebeAddress": "zeebeAddress", + }, + }) + } +} From a5787119514b11ff48e1375c5e33395f17e58287 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:17:11 +0200 Subject: [PATCH 5/7] feat: support credentials on Workers --- charts/zeebe-benchmark/templates/workers.yaml | 26 +++++++ .../golden-credentials-workers.golden.yaml | 69 +++++++++++++++++++ .../test/saas_configuration_test.go | 2 +- 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 charts/zeebe-benchmark/test/golden/golden-credentials-workers.golden.yaml diff --git a/charts/zeebe-benchmark/templates/workers.yaml b/charts/zeebe-benchmark/templates/workers.yaml index cf73765..1a40311 100644 --- a/charts/zeebe-benchmark/templates/workers.yaml +++ b/charts/zeebe-benchmark/templates/workers.yaml @@ -24,7 +24,11 @@ spec: - name: JDK_JAVA_OPTIONS value: >- -Dconfig.override_with_env_vars=true + {{- if $.Values.saas.enabled }} + -Dapp.tls=true + {{- else }} -Dapp.brokerUrl={{ $.Release.Name }}-zeebe-gateway:26500 + {{- end }} -Dzeebe.client.requestTimeout=62000 {{- if $worker.capacity }} -Dapp.worker.capacity={{ $worker.capacity }} @@ -51,6 +55,28 @@ spec: - name: LOG_LEVEL value: {{ $worker.logLevel | quote }} {{- end }} + {{- if $.Values.saas.enabled }} + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: authServer + {{- end }} {{- if $worker.resources }} resources: {{- toYaml $worker.resources | nindent 12 }} diff --git a/charts/zeebe-benchmark/test/golden/golden-credentials-workers.golden.yaml b/charts/zeebe-benchmark/test/golden/golden-credentials-workers.golden.yaml new file mode 100644 index 0000000..2576a93 --- /dev/null +++ b/charts/zeebe-benchmark/test/golden/golden-credentials-workers.golden.yaml @@ -0,0 +1,69 @@ +--- +# Source: zeebe-benchmark/templates/workers.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: benchmark-worker + labels: + app: benchmark-worker +spec: + selector: + matchLabels: + app: benchmark-worker + replicas: 3 + template: + metadata: + labels: + app: benchmark-worker + app.kubernetes.io/component: zeebe-client + spec: + containers: + - name: benchmark-worker + image: "gcr.io/zeebe-io/worker:SNAPSHOT" + imagePullPolicy: Always + env: + - name: JDK_JAVA_OPTIONS + value: >- + -Dconfig.override_with_env_vars=true + -Dapp.tls=true + -Dzeebe.client.requestTimeout=62000 + -Dapp.worker.capacity=60 + -Dapp.worker.pollingDelay=1ms + -Dapp.worker.completionDelay=50ms + -Dapp.worker.workerName="benchmark" + -Dapp.worker.jobType="benchmark-task" + -Dapp.worker.payloadPath="bpmn/big_payload.json" + -Dapp.worker.completionDelay=50ms + -XX:+HeapDumpOnOutOfMemoryError + - name: LOG_LEVEL + value: "WARN" + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: authServer + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 500m + memory: 256Mi + ports: + - containerPort: 9600 + name: "http" \ No newline at end of file diff --git a/charts/zeebe-benchmark/test/saas_configuration_test.go b/charts/zeebe-benchmark/test/saas_configuration_test.go index c4bd113..cc12be9 100644 --- a/charts/zeebe-benchmark/test/saas_configuration_test.go +++ b/charts/zeebe-benchmark/test/saas_configuration_test.go @@ -18,7 +18,7 @@ func TestGoldenCredentials(t *testing.T) { chartPath, err := filepath.Abs("../") require.NoError(t, err) - templateNames := []string{"credentials"} + templateNames := []string{"credentials", "workers"} for _, name := range templateNames { suite.Run(t, &golden.TemplateGoldenTest{ From f89f065ff932f8dc772941eff2dbb4899aeebb03 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:23:35 +0200 Subject: [PATCH 6/7] feat: support credentials on starter --- charts/zeebe-benchmark/templates/starter.yaml | 28 +++++++- .../golden-credentials-starter.golden.yaml | 72 +++++++++++++++++++ .../test/saas_configuration_test.go | 2 +- 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 charts/zeebe-benchmark/test/golden/golden-credentials-starter.golden.yaml diff --git a/charts/zeebe-benchmark/templates/starter.yaml b/charts/zeebe-benchmark/templates/starter.yaml index 9cf5162..57b0d56 100644 --- a/charts/zeebe-benchmark/templates/starter.yaml +++ b/charts/zeebe-benchmark/templates/starter.yaml @@ -23,7 +23,11 @@ spec: - name: JDK_JAVA_OPTIONS value: >- -Dconfig.override_with_env_vars=true - -Dapp.brokerUrl={{ .Release.Name }}-zeebe-gateway:26500 + {{- if $.Values.saas.enabled }} + -Dapp.tls=true + {{- else }} + -Dapp.brokerUrl={{ $.Release.Name }}-zeebe-gateway:26500 + {{- end }} -Dapp.starter.rate={{ .Values.starter.rate }} -Dapp.starter.durationLimit=0 -Dzeebe.client.requestTimeout=62000 @@ -47,6 +51,28 @@ spec: - name: LOG_LEVEL value: {{ .Values.starter.logLevel | quote }} {{- end }} + {{- if $.Values.saas.enabled }} + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: {{ include "zeebe-benchmark.credentials-name" $ }} + key: authServer + {{- end }} envFrom: - configMapRef: name: starter-config diff --git a/charts/zeebe-benchmark/test/golden/golden-credentials-starter.golden.yaml b/charts/zeebe-benchmark/test/golden/golden-credentials-starter.golden.yaml new file mode 100644 index 0000000..3a169dd --- /dev/null +++ b/charts/zeebe-benchmark/test/golden/golden-credentials-starter.golden.yaml @@ -0,0 +1,72 @@ +--- +# Source: zeebe-benchmark/templates/starter.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starter + labels: + app: starter +spec: + selector: + matchLabels: + app: starter + replicas: 1 + template: + metadata: + labels: + app: starter + app.kubernetes.io/component: zeebe-client + spec: + containers: + - name: starter + image: "gcr.io/zeebe-io/starter:SNAPSHOT" + imagePullPolicy: Always + env: + - name: JDK_JAVA_OPTIONS + value: >- + -Dconfig.override_with_env_vars=true + -Dapp.tls=true + -Dapp.starter.rate=150 + -Dapp.starter.durationLimit=0 + -Dzeebe.client.requestTimeout=62000 + -Dapp.starter.processId="benchmark" + -Dapp.starter.bpmnXmlPath="bpmn/one_task.bpmn" + -Dapp.starter.businessKey="businessKey" + -Dapp.starter.payloadPath="bpmn/big_payload.json" + -XX:+HeapDumpOnOutOfMemoryError + - name: LOG_LEVEL + value: "WARN" + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: benchmark-test-credentials + key: authServer + envFrom: + - configMapRef: + name: starter-config + optional: true + resources: + limits: + cpu: 250m + memory: 256Mi + requests: + cpu: 250m + memory: 256Mi + ports: + - containerPort: 9600 + name: "http" \ No newline at end of file diff --git a/charts/zeebe-benchmark/test/saas_configuration_test.go b/charts/zeebe-benchmark/test/saas_configuration_test.go index cc12be9..2908329 100644 --- a/charts/zeebe-benchmark/test/saas_configuration_test.go +++ b/charts/zeebe-benchmark/test/saas_configuration_test.go @@ -18,7 +18,7 @@ func TestGoldenCredentials(t *testing.T) { chartPath, err := filepath.Abs("../") require.NoError(t, err) - templateNames := []string{"credentials", "workers"} + templateNames := []string{"credentials", "workers", "starter"} for _, name := range templateNames { suite.Run(t, &golden.TemplateGoldenTest{ From 9e618b4827e37f78f4ac364baef999a5f4b12c7d Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Fri, 20 Sep 2024 09:27:00 +0200 Subject: [PATCH 7/7] test: verify existing secret behavior --- ...ting-credential-secret-starter.golden.yaml | 72 +++++++++++++++++++ ...ting-credential-secret-workers.golden.yaml | 69 ++++++++++++++++++ .../test/saas_configuration_test.go | 24 +++++++ 3 files changed, 165 insertions(+) create mode 100644 charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-starter.golden.yaml create mode 100644 charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-workers.golden.yaml diff --git a/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-starter.golden.yaml b/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-starter.golden.yaml new file mode 100644 index 0000000..5bfb109 --- /dev/null +++ b/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-starter.golden.yaml @@ -0,0 +1,72 @@ +--- +# Source: zeebe-benchmark/templates/starter.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starter + labels: + app: starter +spec: + selector: + matchLabels: + app: starter + replicas: 1 + template: + metadata: + labels: + app: starter + app.kubernetes.io/component: zeebe-client + spec: + containers: + - name: starter + image: "gcr.io/zeebe-io/starter:SNAPSHOT" + imagePullPolicy: Always + env: + - name: JDK_JAVA_OPTIONS + value: >- + -Dconfig.override_with_env_vars=true + -Dapp.tls=true + -Dapp.starter.rate=150 + -Dapp.starter.durationLimit=0 + -Dzeebe.client.requestTimeout=62000 + -Dapp.starter.processId="benchmark" + -Dapp.starter.bpmnXmlPath="bpmn/one_task.bpmn" + -Dapp.starter.businessKey="businessKey" + -Dapp.starter.payloadPath="bpmn/big_payload.json" + -XX:+HeapDumpOnOutOfMemoryError + - name: LOG_LEVEL + value: "WARN" + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: secret-name + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: secret-name + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: secret-name + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: secret-name + key: authServer + envFrom: + - configMapRef: + name: starter-config + optional: true + resources: + limits: + cpu: 250m + memory: 256Mi + requests: + cpu: 250m + memory: 256Mi + ports: + - containerPort: 9600 + name: "http" \ No newline at end of file diff --git a/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-workers.golden.yaml b/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-workers.golden.yaml new file mode 100644 index 0000000..4d61a54 --- /dev/null +++ b/charts/zeebe-benchmark/test/golden/golden-existing-credential-secret-workers.golden.yaml @@ -0,0 +1,69 @@ +--- +# Source: zeebe-benchmark/templates/workers.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: benchmark-worker + labels: + app: benchmark-worker +spec: + selector: + matchLabels: + app: benchmark-worker + replicas: 3 + template: + metadata: + labels: + app: benchmark-worker + app.kubernetes.io/component: zeebe-client + spec: + containers: + - name: benchmark-worker + image: "gcr.io/zeebe-io/worker:SNAPSHOT" + imagePullPolicy: Always + env: + - name: JDK_JAVA_OPTIONS + value: >- + -Dconfig.override_with_env_vars=true + -Dapp.tls=true + -Dzeebe.client.requestTimeout=62000 + -Dapp.worker.capacity=60 + -Dapp.worker.pollingDelay=1ms + -Dapp.worker.completionDelay=50ms + -Dapp.worker.workerName="benchmark" + -Dapp.worker.jobType="benchmark-task" + -Dapp.worker.payloadPath="bpmn/big_payload.json" + -Dapp.worker.completionDelay=50ms + -XX:+HeapDumpOnOutOfMemoryError + - name: LOG_LEVEL + value: "WARN" + - name: ZEEBE_ADDRESS + valueFrom: + secretKeyRef: + name: secret-name + key: zeebeAddress + - name: ZEEBE_CLIENT_ID + valueFrom: + secretKeyRef: + name: secret-name + key: clientId + - name: ZEEBE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: secret-name + key: clientSecret + - name: ZEEBE_AUTHORIZATION_SERVER_URL + valueFrom: + secretKeyRef: + name: secret-name + key: authServer + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 500m + memory: 256Mi + ports: + - containerPort: 9600 + name: "http" \ No newline at end of file diff --git a/charts/zeebe-benchmark/test/saas_configuration_test.go b/charts/zeebe-benchmark/test/saas_configuration_test.go index 2908329..3616985 100644 --- a/charts/zeebe-benchmark/test/saas_configuration_test.go +++ b/charts/zeebe-benchmark/test/saas_configuration_test.go @@ -37,3 +37,27 @@ func TestGoldenCredentials(t *testing.T) { }) } } + +func TestGoldenCredentialsExistingSecret(t *testing.T) { + // Test which allows to verify also parent chart templates + // This makes sure that properties are correctly set + // OR configurations have been changed + + chartPath, err := filepath.Abs("../") + require.NoError(t, err) + templateNames := []string{"workers", "starter"} + + for _, name := range templateNames { + suite.Run(t, &golden.TemplateGoldenTest{ + ChartPath: chartPath, + Release: "benchmark-test", + Namespace: "benchmark-" + strings.ToLower(random.UniqueId()), + GoldenFileName: "golden-existing-credential-secret-" + name, + Templates: []string{"templates/" + name + ".yaml"}, + SetValues: map[string]string{ + "saas.enabled": "true", + "saas.credentials.existingSecret": "secret-name", + }, + }) + } +}