Merge branch 'dev' into dev1-singlepackage-p3.12

IBM · Oct 11, 2024 · 44d1d62 · 44d1d62
2 parents a81dac0 + fa76928
commit 44d1d62
Show file tree

Hide file tree

Showing 34 changed files with 1,203 additions and 914 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -44,12 +44,6 @@ transform-tests:
 			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;      \
 			continue;			\
 		fi;					\
-		z=$$(echo $${KFP_BLACK_LIST} | grep  $$dir);     \
-		if  [ ! -z "$$z" ]; then		\
-			echo $$dir is black listed. Skipping generation of $$yml;               \
-			continue;			\
-		fi;					\
-		echo Generating $$yml;                  \
 		cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;    \
 	    fi;						\
 	done

diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml
@@ -56,33 +56,26 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
             - name: Test V1 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code2parquet workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code2parquet workflow-test
-                      echo "Run transforms/code/code2parquet completed"
-                   else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                     transform=$(basename "transforms/code/code2parquet")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code2parquet   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code2parquet
+                     fi              
+                  else
+                     echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,32 +92,25 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
             - name: Test V2 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      export KFPv2=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code2parquet workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code2parquet workflow-test
-                      echo "Run transforms/code/code2parquet completed"
+                     transform=$(basename "transforms/code/code2parquet")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code2parquet   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code2parquet
+                     fi              
                   else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
-                  fi
+                     echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi
diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml
@@ -56,33 +56,26 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
             - name: Test V1 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code_quality workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code_quality workflow-test
-                      echo "Run transforms/code/code_quality completed"
-                   else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                     transform=$(basename "transforms/code/code_quality")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code_quality   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code_quality
+                     fi              
+                  else
+                     echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,32 +92,25 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
             - name: Test V2 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      export KFPv2=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code_quality workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code_quality workflow-test
-                      echo "Run transforms/code/code_quality completed"
+                     transform=$(basename "transforms/code/code_quality")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code_quality   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code_quality
+                     fi              
                   else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
-                  fi
+                     echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi
diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml
@@ -56,33 +56,26 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
             - name: Test V1 KFP workflow for transforms/code/header_cleanser
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/header_cleanser workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/header_cleanser workflow-test
-                      echo "Run transforms/code/header_cleanser completed"
-                   else
-                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                     transform=$(basename "transforms/code/header_cleanser")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/header_cleanser   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/header_cleanser
+                     fi              
+                  else
+                     echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
@@ -99,32 +92,25 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
             - name: Test V2 KFP workflow for transforms/code/header_cleanser
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      export KFPv2=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/header_cleanser workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/header_cleanser workflow-test
-                      echo "Run transforms/code/header_cleanser completed"
+                     transform=$(basename "transforms/code/header_cleanser")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/header_cleanser   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/header_cleanser
+                     fi              
                   else
-                      echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
-                  fi
+                     echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi