-
Notifications
You must be signed in to change notification settings - Fork 154
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #694 from revit13/kfp-html2parquet
Kfp workflow for html2parquet- Thanks!
- Loading branch information
Showing
11 changed files
with
449 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| # | ||
| # DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files | ||
| # | ||
| name: Test KFP - transforms/language/html2parquet | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| branches: | ||
| - "dev" | ||
| - "releases/**" | ||
| tags: | ||
| - "*" | ||
| paths: | ||
| - ".make.*" | ||
| - "transforms/.make.workflows" | ||
| - "transforms/language/html2parquet/**" | ||
| - "!kfp/**" # This is tested in separate workflow | ||
| - "!data-processing-lib/**" # This is tested in separate workflow | ||
| - "!**.md" | ||
| - "!**/doc/**" | ||
| - "!**/images/**" | ||
| - "!**.gitignore" | ||
| pull_request: | ||
| branches: | ||
| - "dev" | ||
| - "releases/**" | ||
| paths: | ||
| - ".make.*" | ||
| - "transforms/.make.workflows" | ||
| - "transforms/language/html2parquet/**" | ||
| - "!data-processing-lib/**" # This is tested in separate workflow | ||
| - "!kfp/**" # This is tested in separate workflow | ||
| - "!**.md" | ||
| - "!**/doc/**" | ||
| - "!**/images/**" | ||
| - "!**.gitignore" | ||
|
|
||
| # taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| test-kfp-v1: | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Free up space in github runner | ||
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | ||
| run: | | ||
| df -h | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | ||
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | ||
| df -h | ||
| - name: Import environment variables | ||
| run: | | ||
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | ||
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | ||
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | ||
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | ||
| - name: Test V1 KFP workflow for transforms/language/html2parquet | ||
| timeout-minutes: 120 | ||
| run: | | ||
| KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list) | ||
| if [ -e "transforms/language/html2parquet/Makefile" -a -e "transforms/language/html2parquet/kfp_ray/Makefile" ]; then | ||
| transform=$(basename "transforms/language/html2parquet") | ||
| if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then | ||
| $PWD/scripts/workflow_helper.sh install-tools | ||
| $PWD/scripts/workflow_helper.sh test-workflow transforms/language/html2parquet | ||
| else | ||
| $PWD/scripts/workflow_helper.sh build-workflow transforms/language/html2parquet | ||
| fi | ||
| else | ||
| echo "Skipping transforms/language/html2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile" | ||
| fi | ||
| test-kfp-v2: | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Free up space in github runner | ||
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | ||
| run: | | ||
| df -h | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | ||
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | ||
| df -h | ||
| - name: Import environment variables | ||
| run: | | ||
| cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV | ||
| echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV | ||
| echo "REPOROOT=$PWD" >> $GITHUB_ENV | ||
| echo "PATH=$PATH:/tmp" >> $GITHUB_ENV | ||
| echo "KFPv2=1" >> $GITHUB_ENV | ||
| - name: Test V2 KFP workflow for transforms/language/html2parquet | ||
| timeout-minutes: 120 | ||
| run: | | ||
| KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list) | ||
| if [ -e "transforms/language/html2parquet/Makefile" -a -e "transforms/language/html2parquet/kfp_ray/Makefile" ]; then | ||
| transform=$(basename "transforms/language/html2parquet") | ||
| if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then | ||
| $PWD/scripts/workflow_helper.sh install-tools | ||
| $PWD/scripts/workflow_helper.sh test-workflow transforms/language/html2parquet | ||
| else | ||
| $PWD/scripts/workflow_helper.sh build-workflow transforms/language/html2parquet | ||
| fi | ||
| else | ||
| echo "Skipping transforms/language/html2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| # Adding new KFP workflows | ||
|
|
||
| This README outlines the steps to add a new KFP workflow for a new transform under [transforms](./transforms/) directory. | ||
|
|
||
| 1) Create a new `kfp_ray` directory in the transform directory, similar to [this directory](universal/noop/kfp_ray/). | ||
|
|
||
| 2) Create the workflow and add it to `kfp_ray` directory. It is recommended to use the [pipeline generator](../kfp/pipeline_generator/single-pipeline/) for that. If the workflow was generated using the [pipeline generator](../kfp//pipeline_generator/single-pipeline/) also include `pipeline_definitions.yaml` file used to generate the workflow in the `kfp_ray` directory. | ||
|
|
||
| 3) Add `Makefile` file to `kfp_ray` directory similar to [this Makefile example](./universal/noop/kfp_ray/Makefile). | ||
|
|
||
| 3) Add the path to the transform input directory in the [populate_minio script](../scripts/k8s-setup/populate_minio.sh). This path is used when testing the workflow. | ||
| 4) Create a GitHub Action for the kfp workflow using the `make` command in the [.github/workflows/](../.github/workflows/README.md) directory. | ||
| 5) Update the workflows list in [README.md](../kfp/README.md) file. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| REPOROOT=${CURDIR}/../../../../ | ||
| WORKFLOW_VENV_ACTIVATE=${REPOROOT}/transforms/venv/bin/activate | ||
| include $(REPOROOT)/transforms/.make.workflows | ||
|
|
||
| # Include the common configuration for this transform | ||
| include ../transform.config | ||
|
|
||
| SRC_DIR=${CURDIR}/../ray/ | ||
|
|
||
| PYTHON_WF := $(shell find ./ -name '*_wf.py') | ||
| YAML_WF := $(patsubst %.py, %.yaml, ${PYTHON_WF}) | ||
|
|
||
| workflow-venv: .check_python_version ${WORKFLOW_VENV_ACTIVATE} | ||
|
|
||
| .PHONY: clean | ||
| clean: | ||
| @# Help: Clean up the virtual environment. | ||
| rm -rf ${REPOROOT}/transforms/venv | ||
|
|
||
| venv:: | ||
|
|
||
| build:: | ||
|
|
||
| test:: | ||
|
|
||
| test-src:: | ||
|
|
||
| test-image:: | ||
|
|
||
| publish:: | ||
|
|
||
| image:: | ||
|
|
||
| load-image:: | ||
|
|
||
| .PHONY: workflow-build | ||
| workflow-build: workflow-venv | ||
| $(MAKE) $(YAML_WF) | ||
|
|
||
| .PHONY: workflow-test | ||
| workflow-test: workflow-build | ||
| $(MAKE) .workflows.test-pipeline TRANSFORM_SRC=${SRC_DIR} PIPELINE_FILE=html2parquet_wf.yaml | ||
|
|
||
| .PHONY: workflow-upload | ||
| workflow-upload: | ||
| @for file in $(YAML_WF); do \ | ||
| $(MAKE) .workflows.upload-pipeline PIPELINE_FILE=$$file; \ | ||
| done |
Oops, something went wrong.