diff --git a/api/v1beta1/temporalcluster_types.go b/api/v1beta1/temporalcluster_types.go index 555c405b..7a02aed7 100644 --- a/api/v1beta1/temporalcluster_types.go +++ b/api/v1beta1/temporalcluster_types.go @@ -174,7 +174,8 @@ type DeploymentOverride struct { *ObjectMetaOverride `json:"metadata,omitempty"` // Specification of the desired behavior of the Deployment. // +optional - Spec *DeploymentOverrideSpec `json:"spec,omitempty"` + Spec *DeploymentOverrideSpec `json:"spec,omitempty"` + JSONPatch *apiextensionsv1.JSON `json:"jsonPatch,omitempty"` } // DeploymentOverrideSpec provides the ability to override a Deployment Spec. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 4ae28587..ce8d20cb 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -382,6 +382,11 @@ func (in *DeploymentOverride) DeepCopyInto(out *DeploymentOverride) { *out = new(DeploymentOverrideSpec) (*in).DeepCopyInto(*out) } + if in.JSONPatch != nil { + in, out := &in.JSONPatch, &out.JSONPatch + *out = new(apiextensionsv1.JSON) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentOverride. diff --git a/config/crd/bases/temporal.io_temporalclusters.yaml b/config/crd/bases/temporal.io_temporalclusters.yaml index f4afcd5a..1a5d1d2c 100644 --- a/config/crd/bases/temporal.io_temporalclusters.yaml +++ b/config/crd/bases/temporal.io_temporalclusters.yaml @@ -64,6 +64,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -2910,6 +2912,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3075,6 +3079,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3246,6 +3252,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3411,6 +3419,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3549,6 +3559,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3637,6 +3649,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. @@ -3829,6 +3843,8 @@ spec: deployment: description: Override configuration for the temporal service Deployment. properties: + jsonPatch: + x-kubernetes-preserve-unknown-fields: true metadata: description: |- ObjectMetaOverride provides the ability to override an object metadata. diff --git a/docs/api/v1beta1.md b/docs/api/v1beta1.md index 1a4c6147..27248913 100644 --- a/docs/api/v1beta1.md +++ b/docs/api/v1beta1.md @@ -1400,6 +1400,18 @@ DeploymentOverrideSpec + + +jsonPatch
+ + +k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1.JSON + + + + + + @@ -2388,7 +2400,7 @@ map[string]string override
- + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1.ServiceMonitorSpec @@ -2403,7 +2415,7 @@ All fields can be overwritten except “endpoints”, “selector&rd metricRelabelings
- + []github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1.RelabelConfig diff --git a/docs/features/overrides.md b/docs/features/overrides.md index f45e7415..227c5d59 100644 --- a/docs/features/overrides.md +++ b/docs/features/overrides.md @@ -15,6 +15,8 @@ The API provides you the ability to apply your overrides: - per temporal service (using `spec.services.[frontend|history|matching|worker].overrides`) - for all services (using `spec.services.overrides`) +There are two ways of performing overrides, one is via StrategicPatchMerge and one using RFC6902 JSON patches. You can find examples of both below. If working with certain fields that aren't handled by StrategicPatchMerge properly (i.e., arrays that don't have go struct tags for merging valid for your use case), you may want to consider using JSON patches. + ## Overrides for all services Here is a general example: @@ -210,7 +212,7 @@ spec: value: example.com ``` -### Example: mount an extra volume to the frontend pod +### Example: Mount an extra secret volume to the frontend pod ```yaml apiVersion: temporal.io/v1beta1 @@ -223,18 +225,18 @@ spec: frontend: overrides: deployment: - spec: - template: - spec: - containers: - - name: service - volumeMounts: - - name: extra-volume - mountPath: /etc/extra - volumes: - - name: extra-volume - configMap: - name: extra-config + jsonPatch: + - op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + name: extra-volume + mountPath: /etc/extra + - op: add + path: /spec/template/spec/volumes/- + value: + name: extra-volume + secret: + secretName: test-secret ``` ### Example: Add an environment variable from secretRef to the frontend pod @@ -286,6 +288,29 @@ spec: service: frontend.temporal.temporal.svc.cluster.local ``` +### Example: Add environment variable from a secret to frontend pod +```yaml +apiVersion: temporal.io/v1beta1 +kind: TemporalCluster +metadata: + name: prod +spec: + # [...] + services: + frontend: + overrides: + deployment: + jsonPatch: + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: TEST + valueFrom: + secretKeyRef: + name: test-secret + key: test +``` + Read more in [Strategic Merge Patch](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md#strategic-merge-patch). ## Override UI deployment diff --git a/go.mod b/go.mod index b45a7ebb..ef9971c2 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/alexandrevilain/controller-tools v0.3.0 github.com/cert-manager/cert-manager v1.16.1 github.com/elliotchance/orderedmap/v2 v2.4.0 + github.com/evanphx/json-patch/v5 v5.9.0 github.com/go-logr/logr v1.4.2 github.com/gocql/gocql v1.7.0 github.com/google/uuid v1.6.0 @@ -53,7 +54,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect diff --git a/pkg/kubernetes/overrides.go b/pkg/kubernetes/overrides.go index 1f80ad80..12c908c3 100644 --- a/pkg/kubernetes/overrides.go +++ b/pkg/kubernetes/overrides.go @@ -23,6 +23,7 @@ import ( "github.com/alexandrevilain/temporal-operator/api/v1beta1" "github.com/alexandrevilain/temporal-operator/internal/metadata" + jsonpatch "github.com/evanphx/json-patch/v5" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/strategicpatch" @@ -110,6 +111,24 @@ func ApplyDeploymentOverrides(deployment *appsv1.Deployment, override *v1beta1.D } } + if override.JSONPatch != nil { + patch, err := jsonpatch.DecodePatch(override.JSONPatch.Raw) + if err != nil { + return fmt.Errorf("can't decode json patch: %w", err) + } + + original, err := json.Marshal(deployment) + if err != nil { + return fmt.Errorf("can't marshal deployment spec: %w", err) + } + + patched, err := patch.Apply(original) + if err != nil { + return fmt.Errorf("can't apply json patch: %w", err) + } + return json.Unmarshal(patched, &deployment) + } + return nil } diff --git a/pkg/kubernetes/overrides_test.go b/pkg/kubernetes/overrides_test.go index bf8b1c70..c81d6dc9 100644 --- a/pkg/kubernetes/overrides_test.go +++ b/pkg/kubernetes/overrides_test.go @@ -338,6 +338,166 @@ func TestApplyDeploymentOverrides(t *testing.T) { }, }, }, + "add env var to existing env": { + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Env: []corev1.EnvVar{ + { + Name: "a", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + override: &v1beta1.DeploymentOverride{ + JSONPatch: &apiextensionsv1.JSON{ + Raw: []byte(`[{"op":"add", "path":"/spec/template/spec/containers/0/env/-", "value":{"name":"b","value":"c"}}]`), + }, + }, + expected: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Env: []corev1.EnvVar{ + { + Name: "a", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + { + Name: "b", + Value: "c", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "add secret volume to existing volumes": { + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "a", + ReadOnly: true, + MountPath: "/a", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "a", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, + override: &v1beta1.DeploymentOverride{ + JSONPatch: &apiextensionsv1.JSON{ + Raw: []byte(`[ + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", "value": {"name": "b", "readOnly": true, "mountPath": "/b"}}, + {"op": "add", "path": "/spec/template/spec/volumes/-", "value": {"name": "b", "secret": {"secretName": "test"}}} + ]`), + }, + }, + expected: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "a", + ReadOnly: true, + MountPath: "/a", + }, + { + Name: "b", + ReadOnly: true, + MountPath: "/b", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "a", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test", + }, + }, + }, + }, + { + Name: "b", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + }, + }, + }, + }, + }, + }, } for name, test := range tests { diff --git a/webhooks/temporalcluster_webhook.go b/webhooks/temporalcluster_webhook.go index 4b47f10e..ca02399b 100644 --- a/webhooks/temporalcluster_webhook.go +++ b/webhooks/temporalcluster_webhook.go @@ -28,6 +28,7 @@ import ( "github.com/alexandrevilain/temporal-operator/pkg/version" enumspb "go.temporal.io/api/enums/v1" enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/common/primitives" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" @@ -317,6 +318,46 @@ func (w *TemporalClusterWebhook) validateCluster(cluster *v1beta1.TemporalCluste } } + if cluster.Spec.Services != nil { + overrides := cluster.Spec.Services.Overrides + if overrides != nil && overrides.Deployment != nil && overrides.Deployment.Spec != nil && overrides.Deployment.JSONPatch != nil { + errs = append(errs, + field.Forbidden( + field.NewPath("spec", "services", "overrides", "deployment", "jsonPatch"), + "Can't set JSONPatch when Spec is set on Deployment override", + ), + ) + } + + services := []primitives.ServiceName{ + primitives.FrontendService, + primitives.HistoryService, + primitives.MatchingService, + primitives.WorkerService, + primitives.InternalFrontendService, + } + + for _, service := range services { + spec, err := cluster.Spec.Services.GetServiceSpec(service) + if err != nil { + errs = append(errs, + field.Invalid( + field.NewPath("spec", "services", string(service)), + string(service), + fmt.Sprintf("Invalid service: %s", string(service)), + ), + ) + } + if spec != nil && spec.Overrides != nil && spec.Overrides.Deployment != nil && spec.Overrides.Deployment.Spec != nil && spec.Overrides.Deployment.JSONPatch != nil { + errs = append(errs, + field.Forbidden( + field.NewPath("spec", "services", string(service), "overrides", "deployment", "jsonPatch"), + "Can't set JSONPatch when Spec is set on Deployment override", + ), + ) + } + } + } return warns, errs } diff --git a/webhooks/temporalcluster_webhook_test.go b/webhooks/temporalcluster_webhook_test.go index 66fe3cfa..cea97cc9 100644 --- a/webhooks/temporalcluster_webhook_test.go +++ b/webhooks/temporalcluster_webhook_test.go @@ -26,6 +26,7 @@ import ( "github.com/alexandrevilain/temporal-operator/pkg/version" "github.com/alexandrevilain/temporal-operator/webhooks" "github.com/stretchr/testify/assert" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -302,6 +303,39 @@ func TestValidateCreate(t *testing.T) { }, expectedErr: "TemporalCluster.temporal.io \"fake\" is invalid: spec.persistence.visibilityStore.cassandra: Forbidden: Support for Cassandra as a Visibility database has been removed with Temporal Server v1.24.", }, + "error with override spec and JSON patch": { + object: &v1beta1.TemporalCluster{ + TypeMeta: v1beta1.TemporalClusterTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "fake", + }, + Spec: v1beta1.TemporalClusterSpec{ + Version: version.MustNewVersionFromString("1.18.4"), + Services: &v1beta1.ServicesSpec{ + Overrides: &v1beta1.ServiceSpecOverride{ + Deployment: &v1beta1.DeploymentOverride{ + JSONPatch: &apiextensionsv1.JSON{ + Raw: []byte(`{ "op": "replace", "path": "/spec/replicas", "value": 3 }`), + }, + Spec: &v1beta1.DeploymentOverrideSpec{ + Template: &v1beta1.PodTemplateSpecOverride{ + Spec: &apiextensionsv1.JSON{Raw: []byte(`{ "replicas": 2 }`)}, + }, + }, + }, + }, + }, + }, + }, + wh: &webhooks.TemporalClusterWebhook{ + AvailableAPIs: &discovery.AvailableAPIs{ + Istio: false, + CertManager: false, + PrometheusOperator: false, + }, + }, + expectedErr: "Forbidden: Can't set JSONPatch when Spec is set on Deployment override", + }, } for name, test := range tests {