diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 275fd4688..751b3201f 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -8,7 +8,6 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco # A list that holds transforms that should not be tested with KFP -KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor" transform-tests: $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests @@ -29,13 +28,19 @@ transform-tests: done .transform-kfp-tests: - @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ + @KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list); \ + for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ dir=$$(basename $$i); \ - z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir); \ - if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then \ + yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ + if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then \ + echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \ + continue; \ + fi; \ + z=$$(echo $${KFP_BLACK_LIST} | grep $$dir); \ + if [ ! -z "$$z" ]; then \ + echo $$dir is black listed. Skipping generation of $$yml; \ continue; \ fi; \ - yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ echo Generating $$yml; \ cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ done diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index a2909c55d..09678e937 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -8,7 +8,7 @@ on: - "releases/**" jobs: deploy: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest env: REPO_URL: "https://github.com/${{ github.repository }}" REPO_BRANCH: "dev" diff --git a/.github/workflows/deploy-library.yml b/.github/workflows/deploy-library.yml index 8ec97ed9e..0c2473175 100644 --- a/.github/workflows/deploy-library.yml +++ b/.github/workflows/deploy-library.yml @@ -14,7 +14,7 @@ permissions: jobs: build-package: name: Build Ray data processing libraries - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -30,7 +30,7 @@ jobs: name: Publish packages to test.pypi.org # disabled if: false - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-package steps: @@ -47,7 +47,7 @@ jobs: publish-pypi: name: Publish release to pypi.org - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-package # disabled as of now if: false diff --git a/.github/workflows/deploy-transforms.yml b/.github/workflows/deploy-transforms.yml index 7fe5c8b4d..0f002187d 100644 --- a/.github/workflows/deploy-transforms.yml +++ b/.github/workflows/deploy-transforms.yml @@ -9,7 +9,7 @@ on: jobs: build-images: name: Build and check images - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -23,7 +23,7 @@ jobs: name: Publish packages to quay.io # disabled if: false - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-images steps: diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 41f58e0cb..6de24d4b0 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code2parquet/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code2parquet workflow-test - echo "Run transforms/code/code2parquet completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code2parquet workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + echo "Run transforms/code/code2parquet completed" + else + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code2parquet workflow-test - header_text "Run transforms/code/code2parquet completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code2parquet workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + echo "Run transforms/code/code2parquet completed" + else + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index f8f1654e7..3f83e9856 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 21fa63296..2e22c04a9 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code_quality workflow-test - echo "Run transforms/code/code_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + echo "Run transforms/code/code_quality completed" + else + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code_quality workflow-test - header_text "Run transforms/code/code_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + echo "Run transforms/code/code_quality completed" + else + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index d53c81c61..5a901edbb 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 25f54b528..6cc4727aa 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/header_cleanser/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/header_cleanser/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/header_cleanser workflow-test - echo "Run transforms/code/header_cleanser completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/header_cleanser workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + echo "Run transforms/code/header_cleanser completed" + else + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/header_cleanser workflow-test - header_text "Run transforms/code/header_cleanser completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/header_cleanser workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + echo "Run transforms/code/header_cleanser completed" + else + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 1834f4983..05f09a8c5 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml new file mode 100644 index 000000000..94d662d1d --- /dev/null +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -0,0 +1,130 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/license_select + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - ".make.*" + - "transforms/.make.workflow" + - "transforms/code/license_select/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - ".make.*" + - "transforms/.make.workflow" + - "transforms/code/license_select/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test-kfp-v1: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/license_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + echo "Run transforms/code/license_select completed" + else + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + fi + + test-kfp-v2: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/license_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + echo "Run transforms/code/license_select completed" + else + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-license_select.yml b/.github/workflows/test-code-license_select.yml index ab24b582b..59592c82f 100644 --- a/.github/workflows/test-code-license_select.yml +++ b/.github/workflows/test-code-license_select.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/license_select/**" - "data-processing-lib/**" - "!transforms/code/license_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/license_select/**" - "data-processing-lib/**" - "!transforms/code/license_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 9bd937f46..2c9e3186c 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/malware/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/malware/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/malware workflow-test - echo "Run transforms/code/malware completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/malware workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + echo "Run transforms/code/malware completed" + else + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/malware workflow-test - header_text "Run transforms/code/malware completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/malware workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + echo "Run transforms/code/malware completed" + else + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index debc779d1..44196c62c 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/malware/**" - "data-processing-lib/**" - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/malware/**" - "data-processing-lib/**" - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index bbe257964..c23e0f1ff 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/proglang_select/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/proglang_select/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/proglang_select workflow-test - echo "Run transforms/code/proglang_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/proglang_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + echo "Run transforms/code/proglang_select completed" + else + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/proglang_select workflow-test - header_text "Run transforms/code/proglang_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/proglang_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + echo "Run transforms/code/proglang_select completed" + else + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 36bf6a869..4723e5d3a 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index c26ecda52..57b39f313 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/repo_level_ordering/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/repo_level_ordering/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/repo_level_ordering workflow-test - echo "Run transforms/code/repo_level_ordering completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/repo_level_ordering workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + echo "Run transforms/code/repo_level_ordering completed" + else + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/repo_level_ordering workflow-test - header_text "Run transforms/code/repo_level_ordering completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/repo_level_ordering workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + echo "Run transforms/code/repo_level_ordering completed" + else + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index fe0ee23bb..19ec8daf5 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 434a57238..1003ba643 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "@TARGET_TRANSFORM_DIR@/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "@TARGET_TRANSFORM_DIR@/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C @TARGET_TRANSFORM_DIR@ workflow-test - echo "Run @TARGET_TRANSFORM_DIR@ completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C @TARGET_TRANSFORM_DIR@ workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + echo "Run @TARGET_TRANSFORM_DIR@ completed" + else + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C @TARGET_TRANSFORM_DIR@ workflow-test - header_text "Run @TARGET_TRANSFORM_DIR@ completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C @TARGET_TRANSFORM_DIR@ workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + echo "Run @TARGET_TRANSFORM_DIR@ completed" + else + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index 6719c322e..01deebcfa 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -42,6 +42,11 @@ on: - "!**/images/**" - "!**/.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" @@ -51,7 +56,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -68,7 +73,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -117,7 +122,7 @@ jobs: echo "Run ${transforms[$index]} completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -167,7 +172,7 @@ jobs: header_text "Run ${transforms[$index]} completed" build-kfp-components: needs: [check_if_push_images] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 30 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index fa3ea58ca..ec78512e5 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index e9f678595..1c5237b20 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/doc_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/doc_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/doc_quality workflow-test - echo "Run transforms/language/doc_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/doc_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + echo "Run transforms/language/doc_quality completed" + else + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/doc_quality workflow-test - header_text "Run transforms/language/doc_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/doc_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + echo "Run transforms/language/doc_quality completed" + else + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index dde61e1fa..443c22152 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-html2parquet.yml b/.github/workflows/test-language-html2parquet.yml index 8caf4efd9..e5ef8e510 100644 --- a/.github/workflows/test-language-html2parquet.yml +++ b/.github/workflows/test-language-html2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/html2parquet/**" - "data-processing-lib/**" - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/html2parquet/**" - "data-processing-lib/**" - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index cf3dec397..c6eb179b8 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/lang_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/lang_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/lang_id workflow-test - echo "Run transforms/language/lang_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/lang_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + echo "Run transforms/language/lang_id completed" + else + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/lang_id workflow-test - header_text "Run transforms/language/lang_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/lang_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + echo "Run transforms/language/lang_id completed" + else + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 3b39358c9..7c318a3a1 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index bb523c57e..fbdd81b8e 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 9656a2f24..5ecc80b08 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index d90b76820..8e238dfcf 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/text_encoder/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/text_encoder/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/text_encoder workflow-test - echo "Run transforms/language/text_encoder completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/text_encoder workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + echo "Run transforms/language/text_encoder completed" + else + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/text_encoder workflow-test - header_text "Run transforms/language/text_encoder completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/text_encoder workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + echo "Run transforms/language/text_encoder completed" + else + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index f7622f8e0..d49c1193d 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index be00c2076..5a1cff872 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -25,13 +25,18 @@ on: - "!data-processing-lib/**/doc/**" - "!data-processing-lib/**/.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_images: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -48,7 +53,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-python-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -56,7 +61,7 @@ jobs: run: | make -C data-processing-lib/python DOCKER=docker venv test test-ray-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -64,7 +69,7 @@ jobs: run: | make -C data-processing-lib/ray DOCKER=docker venv test test-spark-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -74,7 +79,7 @@ jobs: test-data-processing-lib-images: needs: [check_if_push_images] if: needs.check_if_push_images.outputs.publish_images == 'true' - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index 2c601bbd5..62c1a187a 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -29,7 +29,7 @@ on: jobs: test-make: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -37,7 +37,7 @@ jobs: run: | make -n clean test build publish set-versions check-transform-test-workflows: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-packaging-python.yml b/.github/workflows/test-packaging-python.yml index 4ee491c8e..e88eeeae2 100644 --- a/.github/workflows/test-packaging-python.yml +++ b/.github/workflows/test-packaging-python.yml @@ -27,7 +27,7 @@ on: jobs: test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-packaging-ray.yml b/.github/workflows/test-packaging-ray.yml index 4b812540c..9dbce3110 100644 --- a/.github/workflows/test-packaging-ray.yml +++ b/.github/workflows/test-packaging-ray.yml @@ -27,7 +27,7 @@ on: jobs: test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index e0966717e..f3907d56a 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index 28c1d8717..8ed1df919 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/doc_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/doc_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/doc_id workflow-test - echo "Run transforms/universal/doc_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/doc_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + echo "Run transforms/universal/doc_id completed" + else + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/doc_id workflow-test - header_text "Run transforms/universal/doc_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/doc_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + echo "Run transforms/universal/doc_id completed" + else + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 66d0283ca..d314f3b25 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 5d3481e30..93408a260 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/ededup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/ededup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/ededup workflow-test - echo "Run transforms/universal/ededup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/ededup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + echo "Run transforms/universal/ededup completed" + else + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/ededup workflow-test - header_text "Run transforms/universal/ededup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/ededup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + echo "Run transforms/universal/ededup completed" + else + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 225c27cc3..8b4034570 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 9d331ed47..27dfc5493 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/fdedup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/fdedup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/fdedup workflow-test - echo "Run transforms/universal/fdedup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/fdedup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + echo "Run transforms/universal/fdedup completed" + else + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/fdedup workflow-test - header_text "Run transforms/universal/fdedup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/fdedup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + echo "Run transforms/universal/fdedup completed" + else + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 356736fca..5f68d4799 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 59ebbde3c..bd2f57229 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/filter/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/filter/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/filter workflow-test - echo "Run transforms/universal/filter completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/filter workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + echo "Run transforms/universal/filter completed" + else + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/filter workflow-test - header_text "Run transforms/universal/filter completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/filter workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + echo "Run transforms/universal/filter completed" + else + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 44858feff..43e936166 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/filter/**" - "data-processing-lib/**" - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/filter/**" - "data-processing-lib/**" - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-hap.yml b/.github/workflows/test-universal-hap.yml index b92e5867c..c845506c1 100644 --- a/.github/workflows/test-universal-hap.yml +++ b/.github/workflows/test-universal-hap.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/hap/**" - "data-processing-lib/**" - "!transforms/universal/hap/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/hap/**" - "data-processing-lib/**" - "!transforms/universal/hap/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index 19c62ab49..01b14e51b 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/noop/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/noop/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/noop workflow-test - echo "Run transforms/universal/noop completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/noop workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + echo "Run transforms/universal/noop completed" + else + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/noop workflow-test - header_text "Run transforms/universal/noop completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/noop workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + echo "Run transforms/universal/noop completed" + else + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index cd72703d1..13e066d58 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/noop/**" - "data-processing-lib/**" - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/noop/**" - "data-processing-lib/**" - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index 3d377922d..e30f7bafa 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/profiler/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/profiler/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/profiler workflow-test - echo "Run transforms/universal/profiler completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/profiler workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + echo "Run transforms/universal/profiler completed" + else + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/profiler workflow-test - header_text "Run transforms/universal/profiler completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/profiler workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + echo "Run transforms/universal/profiler completed" + else + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 50cd8cd26..e018e0ed3 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index fe7377178..630de3c05 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/resize/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/resize/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/resize workflow-test - echo "Run transforms/universal/resize completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/resize workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + echo "Run transforms/universal/resize completed" + else + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/resize workflow-test - header_text "Run transforms/universal/resize completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/resize workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + echo "Run transforms/universal/resize completed" + else + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 99e14b1b8..b3399e5ec 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/resize/**" - "data-processing-lib/**" - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/resize/**" - "data-processing-lib/**" - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index f127db59b..ff13a444c 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/tokenization/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/tokenization/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,10 +36,14 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -51,30 +59,34 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/tokenization workflow-test - echo "Run transforms/universal/tokenization completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/tokenization workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + echo "Run transforms/universal/tokenization completed" + else + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -90,25 +102,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/tokenization workflow-test - header_text "Run transforms/universal/tokenization completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/tokenization workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + echo "Run transforms/universal/tokenization completed" + else + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index e7a620882..ae547c396 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -36,13 +40,18 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -59,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -81,7 +90,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/workflow-manual-run.yml b/.github/workflows/workflow-manual-run.yml index 3c0f37d47..f0f7028b6 100644 --- a/.github/workflows/workflow-manual-run.yml +++ b/.github/workflows/workflow-manual-run.yml @@ -22,7 +22,7 @@ jobs: KFPv2: ${{ github.event.inputs.kfp_v2 }} WORKFLOW_PATH: ${{ github.event.inputs.workflow-path }} DEBUG_MODE: ${{ github.event.inputs.debug }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/scripts/check-workflows.sh b/scripts/check-workflows.sh index 40f4e2615..afc73a886 100755 --- a/scripts/check-workflows.sh +++ b/scripts/check-workflows.sh @@ -1,20 +1,52 @@ #!/bin/bash -# Check that each transform in transforms// has a corresponding -# .github/workflows/test--.yml file. +usage() { +cat << EOF +Check that each transform in transforms// has a corresponding + .github/workflows/test--.yml file and, + .github/workflows/test---kfp.yml file if + there is a kfp_ray directory for the transform, and + the transform is not in the kfp black list. +Options: + -show-kfp-black-list: prints the space separate list of transform + directories (base names) and exits. + -help: show this message. +EOF +} + if [ ! -d transforms ]; then echo Please run this script from the top of the repository exit 1 fi +KFP_BLACK_LIST="doc_chunk pdf2parquet pii_redactor" +while [ $# -ne 0 ]; do + case $1 in + -show-kfp-black-list) echo $KFP_BLACK_LIST; exit 0; + ;; + *help) usage; exit 0; + ;; + *) echo Unrecognized option $1. exit 1 + ;; + esac + shift; +done for i in $(find transforms -maxdepth 2 -mindepth 2 -type d | grep -v venv); do transform=$(basename $i) category=$(dirname $i) category=$(basename $category) - workflow=.github/workflows/test-$category-$transform.yml - if [ ! -e $workflow ]; then - echo Missing $workflow for transform $category/$transform - echo Fix this by running make in the .github/workflows directory - exit 1 + workflows=.github/workflows/test-$category-$transform.yml + is_blacklisted=$(echo $KFP_BLACK_LIST | grep $transform) + if [ -d $i/kfp_ray -a -z "$is_blacklisted" ]; then + workflows="$workflows .github/workflows/test-$category-$transform-kfp.yml" else - echo Verified existence of $workflow - fi + echo KFP workflow for $transform is not expected. + fi + for workflow in $workflows; do + if [ ! -e $workflow ]; then + echo Missing $workflow for transform $category/$transform + echo Fix this by running make in the .github/workflows directory + exit 1 + else + echo Verified existence of $workflow + fi + done done diff --git a/transforms/code/code2parquet/Makefile b/transforms/code/code2parquet/Makefile index bc4077099..027d29644 100644 --- a/transforms/code/code2parquet/Makefile +++ b/transforms/code/code2parquet/Makefile @@ -55,16 +55,25 @@ set-versions: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build \ No newline at end of file + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/code_quality/Makefile b/transforms/code/code_quality/Makefile index 204ea8856..bca6f7e85 100644 --- a/transforms/code/code_quality/Makefile +++ b/transforms/code/code_quality/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/header_cleanser/Makefile b/transforms/code/header_cleanser/Makefile index 204ea8856..bca6f7e85 100644 --- a/transforms/code/header_cleanser/Makefile +++ b/transforms/code/header_cleanser/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/license_select/Makefile b/transforms/code/license_select/Makefile index b19f5c963..04b1cc451 100644 --- a/transforms/code/license_select/Makefile +++ b/transforms/code/license_select/Makefile @@ -47,16 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/malware/Makefile b/transforms/code/malware/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/code/malware/Makefile +++ b/transforms/code/malware/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/proglang_select/Makefile b/transforms/code/proglang_select/Makefile index 9c7c898e4..9e222ee79 100644 --- a/transforms/code/proglang_select/Makefile +++ b/transforms/code/proglang_select/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/repo_level_ordering/Makefile b/transforms/code/repo_level_ordering/Makefile index cebfb4848..04b1cc451 100644 --- a/transforms/code/repo_level_ordering/Makefile +++ b/transforms/code/repo_level_ordering/Makefile @@ -45,15 +45,27 @@ load-image:: @# Help: Recursively make $@ in all subdirs $(MAKE) RULE=$@ .recurse -# kfp implementation is not yet added, so below targets don't do anything. .PHONY: workflow-venv workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/repo_level_ordering/ray/Makefile b/transforms/code/repo_level_ordering/ray/Makefile index 771ed9240..83f8692de 100644 --- a/transforms/code/repo_level_ordering/ray/Makefile +++ b/transforms/code/repo_level_ordering/ray/Makefile @@ -50,3 +50,10 @@ run-s3-sample: .transforms.run-s3-ray-sample minio-start: .minio-start load-image:: .transforms.load-image + +kind-load-image:: .transforms.kind-load-image + +docker-load-image: .defaults.docker-load-image + +docker-save-image: .defaults.docker-save-image + diff --git a/transforms/language/doc_chunk/Makefile b/transforms/language/doc_chunk/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/language/doc_chunk/Makefile +++ b/transforms/language/doc_chunk/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/doc_quality/Makefile b/transforms/language/doc_quality/Makefile index 5cded280a..a3f1865be 100644 --- a/transforms/language/doc_quality/Makefile +++ b/transforms/language/doc_quality/Makefile @@ -47,16 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build \ No newline at end of file + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/html2parquet/Makefile b/transforms/language/html2parquet/Makefile index 017eb23b4..bca6f7e85 100644 --- a/transforms/language/html2parquet/Makefile +++ b/transforms/language/html2parquet/Makefile @@ -55,12 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/lang_id/Makefile b/transforms/language/lang_id/Makefile index 2967ceb67..af4a86873 100644 --- a/transforms/language/lang_id/Makefile +++ b/transforms/language/lang_id/Makefile @@ -60,16 +60,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/pdf2parquet/Makefile b/transforms/language/pdf2parquet/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/language/pdf2parquet/Makefile +++ b/transforms/language/pdf2parquet/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/pii_redactor/Makefile.disable b/transforms/language/pii_redactor/Makefile.disable index f9f635f07..8764d0dc2 100644 --- a/transforms/language/pii_redactor/Makefile.disable +++ b/transforms/language/pii_redactor/Makefile.disable @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/text_encoder/Makefile b/transforms/language/text_encoder/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/language/text_encoder/Makefile +++ b/transforms/language/text_encoder/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/doc_id/Makefile b/transforms/universal/doc_id/Makefile index 7ebb1b8e1..be26d3bf4 100644 --- a/transforms/universal/doc_id/Makefile +++ b/transforms/universal/doc_id/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/ededup/Makefile b/transforms/universal/ededup/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/ededup/Makefile +++ b/transforms/universal/ededup/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/fdedup/Makefile b/transforms/universal/fdedup/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/fdedup/Makefile +++ b/transforms/universal/fdedup/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/filter/Makefile b/transforms/universal/filter/Makefile index 9c7c898e4..9e222ee79 100644 --- a/transforms/universal/filter/Makefile +++ b/transforms/universal/filter/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/hap/Makefile b/transforms/universal/hap/Makefile index 017eb23b4..cdb34d72e 100644 --- a/transforms/universal/hap/Makefile +++ b/transforms/universal/hap/Makefile @@ -55,12 +55,24 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi .PHONY: workflow-upload workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/noop/Makefile b/transforms/universal/noop/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/noop/Makefile +++ b/transforms/universal/noop/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/profiler/Makefile b/transforms/universal/profiler/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/profiler/Makefile +++ b/transforms/universal/profiler/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/resize/Makefile b/transforms/universal/resize/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/resize/Makefile +++ b/transforms/universal/resize/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/tokenization/Makefile b/transforms/universal/tokenization/Makefile index 05d3c3111..bca6f7e85 100644 --- a/transforms/universal/tokenization/Makefile +++ b/transforms/universal/tokenization/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi +