Merge branch 'IBM:dev' into dev-pankaj

IBM · Oct 15, 2024 · 892283b · 892283b
2 parents 4f0bdd4 + 97810af
commit 892283b
Show file tree

Hide file tree

Showing 246 changed files with 3,506 additions and 2,081 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -44,12 +44,6 @@ transform-tests:
 			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;      \
 			continue;			\
 		fi;					\
-		z=$$(echo $${KFP_BLACK_LIST} | grep  $$dir);     \
-		if  [ ! -z "$$z" ]; then		\
-			echo $$dir is black listed. Skipping generation of $$yml;               \
-			continue;			\
-		fi;					\
-		echo Generating $$yml;                  \
 		cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;    \
 	    fi;						\
 	done

diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
@@ -8,7 +8,7 @@ on:
             - "releases/**"
 jobs:
     deploy:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         env:
             REPO_URL: "https://github.com/${{ github.repository }}"
             REPO_BRANCH: "dev"

diff --git a/.github/workflows/deploy-library.yml b/.github/workflows/deploy-library.yml
@@ -14,7 +14,7 @@ permissions:
 jobs:
     build-package:
         name: Build Ray data processing libraries
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -30,7 +30,7 @@ jobs:
         name: Publish packages to test.pypi.org
         # disabled
         if: false
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         needs: build-package
 
         steps:
@@ -47,7 +47,7 @@ jobs:
 
     publish-pypi:
         name: Publish release to pypi.org
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         needs: build-package
         # disabled as of now
         if: false

diff --git a/.github/workflows/deploy-transforms.yml b/.github/workflows/deploy-transforms.yml
@@ -9,7 +9,7 @@ on:
 jobs:
     build-images:
         name: Build and check images
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -23,7 +23,7 @@ jobs:
         name: Publish packages to quay.io
         # disabled
         if: false
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         needs: build-images
 
         steps:

diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml
@@ -43,7 +43,7 @@ concurrency:
 
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -56,37 +56,30 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
             - name: Test V1 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code2parquet workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code2parquet workflow-test
-                      echo "Run transforms/code/code2parquet completed"
-                   else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                     transform=$(basename "transforms/code/code2parquet")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code2parquet   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code2parquet
+                     fi              
+                  else
+                     echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -99,32 +92,25 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
             - name: Test V2 KFP workflow for transforms/code/code2parquet
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      export KFPv2=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code2parquet workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code2parquet workflow-test
-                      echo "Run transforms/code/code2parquet completed"
+                     transform=$(basename "transforms/code/code2parquet")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code2parquet   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code2parquet
+                     fi              
                   else
-                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
-                  fi
+                     echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi
diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml
@@ -51,7 +51,7 @@ jobs:
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -68,7 +68,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -90,7 +90,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}

diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml
@@ -43,7 +43,7 @@ concurrency:
 
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -56,37 +56,30 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
             - name: Test V1 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code_quality workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code_quality workflow-test
-                      echo "Run transforms/code/code_quality completed"
-                   else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                     transform=$(basename "transforms/code/code_quality")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code_quality   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code_quality
+                     fi              
+                  else
+                     echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
                    fi
 
     test-kfp-v2:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -99,32 +92,25 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
             - name: Test V2 KFP workflow for transforms/code/code_quality
               timeout-minutes: 120
               run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
                   if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
-                      export REPOROOT=$PWD
-                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                      source $K8S_SETUP_SCRIPTS/requirements.env
-                      export PATH=$PATH:/tmp/
-                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                      chmod 777 /tmp/kind
-                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                      chmod 700 /tmp/get_helm.sh
-                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                      chmod 777 /tmp/helm
-                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                      chmod 777 /tmp/kubectl
-                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                      chmod +x /tmp/mc
-                      export DEPLOY_KUBEFLOW=1
-                      export KFPv2=1
-                      make -C $K8S_SETUP_SCRIPTS setup
-                      #make -C kfp/kfp_support_lib test
-                      make -C transforms/code/code_quality workflow-build
-                      source $K8S_SETUP_SCRIPTS/common.sh
-                      make -C transforms/code/code_quality workflow-test
-                      echo "Run transforms/code/code_quality completed"
+                     transform=$(basename "transforms/code/code_quality")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/code/code_quality   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/code/code_quality
+                     fi              
                   else
-                      echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
-                  fi
+                     echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi
diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml
@@ -51,7 +51,7 @@ jobs:
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -68,7 +68,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -90,7 +90,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-22.04
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}