chore: Add ability to mount self-signed certs to kfp (#10849)

* Add ability to mount self-signed certs to kfp This update allows CA bundles to be mounted to the launcher/executor pods since those make external connections to object store, which can be behind self signed certs. Detailed Changes: - Added `REQUESTS_CA_BUNDLE` to the environment variables. This is necessary because many Python-based libraries (e.g., requests) utilize this environment variable for SSL/TLS certificate verification. Notably, even though Boto3 is documented to use `AWS_CA_BUNDLE`, tests have shown that it only respects `REQUESTS_CA_BUNDLE`. Reference: https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification and aws/aws-cli#3425. - Configured `AWS_CA_BUNDLE` for AWS CLI and related utilities to ensure AWS services utilize our custom CA bundle for SSL/TLS. - Set up `SSL_CERT_FILE` environment variable for OpenSSL's default certificate file. This setting is important as the `SSL_CERT_DIR` path adjustments had inconsistent results across different environments, as discussed in OpenSSL documentation: https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html Signed-off-by: ddalvi <[email protected]> Co-authored-by: Vani Haripriya <[email protected]> Co-authored-by: Humair Khan <[email protected]> Signed-off-by: ddalvi <[email protected]> * Add unit test to check the certificate mounting Signed-off-by: ddalvi <[email protected]> --------- Signed-off-by: ddalvi <[email protected]> Co-authored-by: Vani Haripriya <[email protected]> Co-authored-by: Humair Khan <[email protected]>
kubeflow · Jun 27, 2024 · 29b7d2f · 29b7d2f
1 parent bdc3bb1
commit 29b7d2f
Show file tree

Hide file tree

Showing 2 changed files with 130 additions and 0 deletions.
diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go
@@ -15,7 +15,9 @@
 package argocompiler
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
 	wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	"github.com/golang/protobuf/jsonpb"
@@ -27,6 +29,7 @@ import (
 
 const (
 	volumeNameKFPLauncher    = "kfp-launcher"
+	volumeNameCABundle       = "ca-bundle"
 	DefaultLauncherImage     = "gcr.io/ml-pipeline/kfp-launcher@sha256:8fe5e6e4718f20b021736022ad3741ddf2abd82aa58c86ae13e89736fdc3f08f"
 	LauncherImageEnvVar      = "V2_LAUNCHER_IMAGE"
 	DefaultDriverImage       = "gcr.io/ml-pipeline/kfp-driver@sha256:3c0665cd36aa87e4359a4c8b6271dcba5bdd817815cd0496ed12eb5dde5fd2ec"
@@ -361,6 +364,59 @@ func (c *workflowCompiler) addContainerExecutorTemplate(refName string) string {
 			extendPodMetadata(&executor.Metadata, k8sExecCfg)
 		}
 	}
+	caBundleCfgMapName := os.Getenv("EXECUTOR_CABUNDLE_CONFIGMAP_NAME")
+	caBundleCfgMapKey := os.Getenv("EXECUTOR_CABUNDLE_CONFIGMAP_KEY")
+	caBundleMountPath := os.Getenv("EXECUTOR_CABUNDLE_MOUNTPATH")
+	if caBundleCfgMapName != "" && caBundleCfgMapKey != "" {
+		caFile := fmt.Sprintf("%s/%s", caBundleMountPath, caBundleCfgMapKey)
+		var certDirectories = []string{
+			caBundleMountPath,
+			"/etc/ssl/certs",
+			"/etc/pki/tls/certs",
+		}
+		// Add to REQUESTS_CA_BUNDLE for python request library.
+		executor.Container.Env = append(executor.Container.Env, k8score.EnvVar{
+			Name:  "REQUESTS_CA_BUNDLE",
+			Value: caFile,
+		})
+		// For AWS utilities like cli, and packages.
+		executor.Container.Env = append(executor.Container.Env, k8score.EnvVar{
+			Name:  "AWS_CA_BUNDLE",
+			Value: caFile,
+		})
+		// OpenSSL default cert file env variable.
+		// https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html
+		executor.Container.Env = append(executor.Container.Env, k8score.EnvVar{
+			Name:  "SSL_CERT_FILE",
+			Value: caFile,
+		})
+		sslCertDir := strings.Join(certDirectories, ":")
+		executor.Container.Env = append(executor.Container.Env, k8score.EnvVar{
+			Name:  "SSL_CERT_DIR",
+			Value: sslCertDir,
+		})
+		volume := k8score.Volume{
+			Name: volumeNameCABundle,
+			VolumeSource: k8score.VolumeSource{
+				ConfigMap: &k8score.ConfigMapVolumeSource{
+					LocalObjectReference: k8score.LocalObjectReference{
+						Name: caBundleCfgMapName,
+					},
+				},
+			},
+		}
+
+		executor.Volumes = append(executor.Volumes, volume)
+
+		volumeMount := k8score.VolumeMount{
+			Name:      volumeNameCABundle,
+			MountPath: caFile,
+			SubPath:   caBundleCfgMapKey,
+		}
+
+		executor.Container.VolumeMounts = append(executor.Container.VolumeMounts, volumeMount)
+
+	}
 	c.templates[nameContainerImpl] = executor
 	c.wf.Spec.Templates = append(c.wf.Spec.Templates, *container, *executor)
 	return nameContainerExecutor

diff --git a/backend/src/v2/compiler/argocompiler/container_test.go b/backend/src/v2/compiler/argocompiler/container_test.go
@@ -15,13 +15,87 @@
 package argocompiler
 
 import (
+	"os"
 	"testing"
 
 	wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	"github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform"
 	"github.com/stretchr/testify/assert"
 )
 
+func TestAddContainerExecutorTemplate(t *testing.T) {
+	tests := []struct {
+		name                  string
+		configMapName         string
+		configMapKey          string
+		mountPath             string
+		expectedVolumeName    string
+		expectedConfigMapName string
+		expectedMountPath     string
+	}{
+		{
+			name:                  "Test with valid settings",
+			configMapName:         "kube-root-ca.crt",
+			configMapKey:          "ca.crt",
+			mountPath:             "/etc/ssl/custom",
+			expectedVolumeName:    "ca-bundle",
+			expectedConfigMapName: "kube-root-ca.crt",
+			expectedMountPath:     "/etc/ssl/custom/ca.crt",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			os.Setenv("EXECUTOR_CABUNDLE_CONFIGMAP_NAME", tt.configMapName)
+			os.Setenv("EXECUTOR_CABUNDLE_CONFIGMAP_KEY", tt.configMapKey)
+			os.Setenv("EXECUTOR_CABUNDLE_MOUNTPATH", tt.mountPath)
+
+			c := &workflowCompiler{
+				templates: make(map[string]*wfapi.Template),
+				wf: &wfapi.Workflow{
+					Spec: wfapi.WorkflowSpec{
+						Templates: []wfapi.Template{},
+					},
+				},
+			}
+
+			c.addContainerExecutorTemplate("test-ref")
+			assert.NotEmpty(t, "system-container-impl", "Template name should not be empty")
+
+			executorTemplate, exists := c.templates["system-container-impl"]
+			assert.True(t, exists, "Template should exist with the returned name")
+			assert.NotNil(t, executorTemplate, "Executor template should not be nil")
+
+			foundVolume := false
+			for _, volume := range executorTemplate.Volumes {
+				if volume.Name == tt.expectedVolumeName {
+					foundVolume = true
+					assert.Equal(t, tt.expectedConfigMapName, volume.VolumeSource.ConfigMap.Name, "ConfigMap name should match")
+					break
+				}
+			}
+			assert.True(t, foundVolume, "CA bundle volume should be included in the template")
+
+			foundVolumeMount := false
+			if executorTemplate.Container != nil {
+				for _, mount := range executorTemplate.Container.VolumeMounts {
+					if mount.Name == tt.expectedVolumeName && mount.MountPath == tt.expectedMountPath {
+						foundVolumeMount = true
+						break
+					}
+				}
+			}
+			assert.True(t, foundVolumeMount, "CA bundle volume mount should be included in the container")
+		})
+	}
+	defer func() {
+		os.Unsetenv("EXECUTOR_CABUNDLE_CONFIGMAP_NAME")
+		os.Unsetenv("EXECUTOR_CABUNDLE_CONFIGMAP_KEY")
+		os.Unsetenv("EXECUTOR_CABUNDLE_MOUNTPATH")
+	}()
+
+}
+
 func Test_extendPodMetadata(t *testing.T) {
 	tests := []struct {
 		name                     string