diff --git a/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py b/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py index e98b79151..3a01eca6d 100644 --- a/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py +++ b/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py @@ -10,6 +10,7 @@ # limitations under the License. ################################################################################ +import os import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl @@ -19,7 +20,7 @@ # empty comment to triigger pre-commit # Components # For every sub workflow we need a separate components, that knows about this subworkflow. -component_spec_path = "../../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", "../../../../../kfp/kfp_ray_components/") run_code_to_parquet_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") run_code_quality_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") run_malware_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") diff --git a/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py b/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py index a27eeff17..fadda6a57 100644 --- a/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py +++ b/examples/kfp-pipelines/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ +import os import kfp.compiler as compiler import kfp.components as comp @@ -17,7 +18,7 @@ # Components # path to kfp component specifications files -component_spec_path = "../../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", "../../../../../kfp/kfp_ray_components/") # For every sub workflow we need a separate components, that knows about this subworkflow. run_doc_id_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") run_exact_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") diff --git a/examples/kfp-pipelines/superworkflows/ray/kfp_v2/Makefile b/examples/kfp-pipelines/superworkflows/ray/kfp_v2/Makefile index db8974494..8227365b7 100644 --- a/examples/kfp-pipelines/superworkflows/ray/kfp_v2/Makefile +++ b/examples/kfp-pipelines/superworkflows/ray/kfp_v2/Makefile @@ -12,7 +12,7 @@ worflow-clean:: .workflows.clean .PHONY: workflow-build workflow-build: workflow-venv @for file in $(YAML_WF); do \ - $(MAKE) $$file; \ + $(MAKE) KFP_COMPONENT_SPEC_PATH=${REPOROOT}/kfp/kfp_ray_components/ $$file; \ done workflow-test:: diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/__init__.py b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/__init__.py index e13f8e8a7..aa845df5f 100644 --- a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/__init__.py +++ b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/__init__.py @@ -1 +1,6 @@ -from workflow_support.compile_utils.component import ONE_HOUR_SEC, ONE_DAY_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils.component import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) \ No newline at end of file diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/component.py b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/component.py index 1b1c8ba24..85326b739 100644 --- a/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/component.py +++ b/kfp/kfp_support_lib/kfp_v1_workflow_support/src/workflow_support/compile_utils/component.py @@ -28,6 +28,9 @@ logger = get_logger(__name__) +# Default path for KFP component specification files +DEFAULT_KFP_COMPONENT_SPEC_PATH = "../../../../kfp/kfp_ray_components/" + ONE_HOUR_SEC = 60 * 60 ONE_DAY_SEC = ONE_HOUR_SEC * 24 ONE_WEEK_SEC = ONE_DAY_SEC * 7 diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/__init__.py b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/__init__.py index e13f8e8a7..aa845df5f 100644 --- a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/__init__.py +++ b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/__init__.py @@ -1 +1,6 @@ -from workflow_support.compile_utils.component import ONE_HOUR_SEC, ONE_DAY_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils.component import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) \ No newline at end of file diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/component.py b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/component.py index fe4e46669..e68095de2 100644 --- a/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/component.py +++ b/kfp/kfp_support_lib/kfp_v2_workflow_support/src/workflow_support/compile_utils/component.py @@ -25,6 +25,9 @@ RUN_NAME = "KFP_RUN_NAME" +# Default path for KFP component specification files +DEFAULT_KFP_COMPONENT_SPEC_PATH = "../../../../kfp/kfp_ray_components/" + ONE_HOUR_SEC = 60 * 60 ONE_DAY_SEC = ONE_HOUR_SEC * 24 ONE_WEEK_SEC = ONE_DAY_SEC * 7 diff --git a/kfp/pipeline_generator/single-pipeline/example/pipeline_definitions.yaml b/kfp/pipeline_generator/single-pipeline/example/pipeline_definitions.yaml index 540960cb9..d703d36ca 100644 --- a/kfp/pipeline_generator/single-pipeline/example/pipeline_definitions.yaml +++ b/kfp/pipeline_generator/single-pipeline/example/pipeline_definitions.yaml @@ -6,7 +6,6 @@ pipeline_parameters: multi_s3: False compute_func_name: "" compute_func_import: "" - component_spec_path: "" pipeline_common_input_parameters_values: kfp_base_image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" diff --git a/kfp/pipeline_generator/single-pipeline/pipeline_generator.py b/kfp/pipeline_generator/single-pipeline/pipeline_generator.py index 225fbaa1b..8872b6d24 100644 --- a/kfp/pipeline_generator/single-pipeline/pipeline_generator.py +++ b/kfp/pipeline_generator/single-pipeline/pipeline_generator.py @@ -47,15 +47,10 @@ common_input_params_values = pipeline_definitions[PIPELINE_COMMON_INPUT_PARAMETERS_VALUES] pipeline_transform_input_parameters = pipeline_definitions[PIPELINE_TRANSFORM_INPUT_PARAMETERS] - component_spec_path = pipeline_parameters.get("component_spec_path", "") - if component_spec_path == "": - component_spec_path = "../../../../kfp/kfp_ray_components/" - content = template.render( transform_image=common_input_params_values["transform_image"], script_name=pipeline_parameters["script_name"], kfp_base_image=common_input_params_values["kfp_base_image"], - component_spec_path=component_spec_path, pipeline_arguments=pipeline_transform_input_parameters["pipeline_arguments"], pipeline_name=pipeline_parameters[NAME], pipeline_description=pipeline_parameters["description"], diff --git a/kfp/pipeline_generator/single-pipeline/templates/simple_pipeline.py b/kfp/pipeline_generator/single-pipeline/templates/simple_pipeline.py index e5d5f7688..ce7657a5c 100644 --- a/kfp/pipeline_generator/single-pipeline/templates/simple_pipeline.py +++ b/kfp/pipeline_generator/single-pipeline/templates/simple_pipeline.py @@ -15,7 +15,12 @@ import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "{{ transform_image }}" @@ -27,7 +32,7 @@ base_kfp_image = "{{ kfp_base_image }}" # path to kfp component specifications files -component_spec_path = "{{ component_spec_path }}" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/kfp/pipeline_generator/superpipeline/generated/sample-super-kubeflow-pipeline_wf.py b/kfp/pipeline_generator/superpipeline/generated/sample-super-kubeflow-pipeline_wf.py index 92f97763d..b8a076409 100644 --- a/kfp/pipeline_generator/superpipeline/generated/sample-super-kubeflow-pipeline_wf.py +++ b/kfp/pipeline_generator/superpipeline/generated/sample-super-kubeflow-pipeline_wf.py @@ -1,13 +1,19 @@ # NOTE: This file is auto generated by Pipeline Generator. +import os import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # path to kfp component specifications files -component_spec_path = "../../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) # For every sub workflow we need a separate components, that knows about this subworkflow. run_doc_id_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") run_ededup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") diff --git a/kfp/pipeline_generator/superpipeline/super_pipeline_generator.py b/kfp/pipeline_generator/superpipeline/super_pipeline_generator.py index 27c124362..f5facb927 100644 --- a/kfp/pipeline_generator/superpipeline/super_pipeline_generator.py +++ b/kfp/pipeline_generator/superpipeline/super_pipeline_generator.py @@ -11,7 +11,7 @@ ################################################################################ import yaml - +import os PRE_COMMIT = "../pre-commit-config.yaml" PIPELINE_TEMPLATE_FILE = "template_superpipeline.py" @@ -69,10 +69,6 @@ def get_generic_params(params, prefix="") -> str: pipeline_tasks = pipeline_definitions[PIPELINE_TASKS] common_input_params = pipeline_definitions[COMMON_INPUT_PARAMETERS] - component_spec_path = pipeline_metadata.get("component_spec_path", "") - if component_spec_path == "": - component_spec_path = "../../../../../kfp/kfp_ray_components/" - for task in pipeline_tasks: task_name = task["name"] task_pipeline_name = task["pipeline_name"] @@ -144,7 +140,6 @@ def get_generic_params(params, prefix="") -> str: superpipeline_name=pipeline_metadata[NAME], superpipeline_description=pipeline_metadata[DESCRIPTION], sub_workflows_components=pipeline_definitions[PIPELINE_TASKS], - component_spec_path=component_spec_path, p1_parameters=pipeline_definitions[PIPELINE_TASKS], add_p2_parameters=common_input_params, sub_workflows_parameters=sub_workflows_parameters, diff --git a/kfp/pipeline_generator/superpipeline/templates/superpipeline.ptmpl b/kfp/pipeline_generator/superpipeline/templates/superpipeline.ptmpl index 6f00e0974..c1a519626 100644 --- a/kfp/pipeline_generator/superpipeline/templates/superpipeline.ptmpl +++ b/kfp/pipeline_generator/superpipeline/templates/superpipeline.ptmpl @@ -1,13 +1,19 @@ # NOTE: This file is auto generated by Pipeline Generator. +import os import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # path to kfp component specifications files -component_spec_path = "__component_spec_path__" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) # For every sub workflow we need a separate components, that knows about this subworkflow. __sub_workflows_components__ diff --git a/kfp/pipeline_generator/superpipeline/templates/template_superpipeline.py b/kfp/pipeline_generator/superpipeline/templates/template_superpipeline.py index 7d69b0143..8051716d8 100644 --- a/kfp/pipeline_generator/superpipeline/templates/template_superpipeline.py +++ b/kfp/pipeline_generator/superpipeline/templates/template_superpipeline.py @@ -1,13 +1,20 @@ # NOTE: This file is auto generated by Pipeline Generator. +import os + import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # path to kfp component specifications files -component_spec_path = "{{ component_spec_path }}" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) # For every sub workflow we need a separate components, that knows about this subworkflow. {%- for component in sub_workflows_components %} run_{{ component.name }}_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml") diff --git a/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py b/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py index f3f491e4b..e506ab5b3 100644 --- a/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py +++ b/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py @@ -15,7 +15,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -28,7 +33,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/code/code_quality/kfp_ray/code_quality_wf.py b/transforms/code/code_quality/kfp_ray/code_quality_wf.py index 6a4ccec1b..f37fb5870 100644 --- a/transforms/code/code_quality/kfp_ray/code_quality_wf.py +++ b/transforms/code/code_quality/kfp_ray/code_quality_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py b/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py index 1e244ab98..6fdf1862a 100644 --- a/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py +++ b/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/code/license_select/kfp_ray/license_select_wf.py b/transforms/code/license_select/kfp_ray/license_select_wf.py index 7dba0d9d1..7c10b1c34 100644 --- a/transforms/code/license_select/kfp_ray/license_select_wf.py +++ b/transforms/code/license_select/kfp_ray/license_select_wf.py @@ -15,7 +15,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -28,7 +33,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/code/malware/kfp_ray/malware_wf.py b/transforms/code/malware/kfp_ray/malware_wf.py index bede80b88..30525e870 100644 --- a/transforms/code/malware/kfp_ray/malware_wf.py +++ b/transforms/code/malware/kfp_ray/malware_wf.py @@ -15,7 +15,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py b/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py index 11f001bfa..f1b271d3c 100644 --- a/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py +++ b/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py @@ -15,7 +15,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As diff --git a/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py b/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py index 38a829fab..47388f394 100644 --- a/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py +++ b/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py @@ -15,7 +15,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/repo_level_order-ray:latest" @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/doc_chunk/kfp_ray/doc_chunk_multiple_wf.py b/transforms/language/doc_chunk/kfp_ray/doc_chunk_multiple_wf.py index 63be2c878..62161be6d 100644 --- a/transforms/language/doc_chunk/kfp_ray/doc_chunk_multiple_wf.py +++ b/transforms/language/doc_chunk/kfp_ray/doc_chunk_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/doc_chunk-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/doc_chunk/kfp_ray/doc_chunk_wf.py b/transforms/language/doc_chunk/kfp_ray/doc_chunk_wf.py index d8b0a49af..618c11d68 100644 --- a/transforms/language/doc_chunk/kfp_ray/doc_chunk_wf.py +++ b/transforms/language/doc_chunk/kfp_ray/doc_chunk_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/doc_chunk-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/doc_quality/kfp_ray/doc_quality_multiple_wf.py b/transforms/language/doc_quality/kfp_ray/doc_quality_multiple_wf.py index 3890cd64e..4a2d9de1d 100644 --- a/transforms/language/doc_quality/kfp_ray/doc_quality_multiple_wf.py +++ b/transforms/language/doc_quality/kfp_ray/doc_quality_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/doc_quality-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different tranforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/doc_quality/kfp_ray/doc_quality_wf.py b/transforms/language/doc_quality/kfp_ray/doc_quality_wf.py index be7a09577..e26efe832 100644 --- a/transforms/language/doc_quality/kfp_ray/doc_quality_wf.py +++ b/transforms/language/doc_quality/kfp_ray/doc_quality_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/doc_quality-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different tranforms might need different implementations. As diff --git a/transforms/language/html2parquet/kfp_ray/html2parquet_wf.py b/transforms/language/html2parquet/kfp_ray/html2parquet_wf.py index 1546c2d30..b6f5dff19 100644 --- a/transforms/language/html2parquet/kfp_ray/html2parquet_wf.py +++ b/transforms/language/html2parquet/kfp_ray/html2parquet_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/html2parquet-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py b/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py index 96d1f11ce..941d32627 100644 --- a/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py +++ b/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different tranforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/lang_id/kfp_ray/lang_id_wf.py b/transforms/language/lang_id/kfp_ray/lang_id_wf.py index 19f293a5d..fa4debbe3 100644 --- a/transforms/language/lang_id/kfp_ray/lang_id_wf.py +++ b/transforms/language/lang_id/kfp_ray/lang_id_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different tranforms might need different implementations. As diff --git a/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_multiple_wf.py b/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_multiple_wf.py index 220a72407..91d40567e 100644 --- a/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_multiple_wf.py +++ b/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/pdf2parquet-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_wf.py b/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_wf.py index 42a3a16a1..4dab7d4af 100644 --- a/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_wf.py +++ b/transforms/language/pdf2parquet/kfp_ray/pdf2parquet_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/pdf2parquet-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/pii_redactor/kfp_ray/pii_redactor_wf.py b/transforms/language/pii_redactor/kfp_ray/pii_redactor_wf.py index d457470ba..b01d51fdf 100644 --- a/transforms/language/pii_redactor/kfp_ray/pii_redactor_wf.py +++ b/transforms/language/pii_redactor/kfp_ray/pii_redactor_wf.py @@ -13,7 +13,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/pii-redactor-ray:latest" @@ -25,7 +30,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/text_encoder/kfp_ray/text_encoder_multiple_wf.py b/transforms/language/text_encoder/kfp_ray/text_encoder_multiple_wf.py index 13fc9eef6..2005ee163 100644 --- a/transforms/language/text_encoder/kfp_ray/text_encoder_multiple_wf.py +++ b/transforms/language/text_encoder/kfp_ray/text_encoder_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/text_encoder-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/language/text_encoder/kfp_ray/text_encoder_wf.py b/transforms/language/text_encoder/kfp_ray/text_encoder_wf.py index a95343874..aa63e23f8 100644 --- a/transforms/language/text_encoder/kfp_ray/text_encoder_wf.py +++ b/transforms/language/text_encoder/kfp_ray/text_encoder_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/text_encoder-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/doc_id/kfp_ray/doc_id_wf.py b/transforms/universal/doc_id/kfp_ray/doc_id_wf.py index 6cff86b85..c5d4cac6d 100644 --- a/transforms/universal/doc_id/kfp_ray/doc_id_wf.py +++ b/transforms/universal/doc_id/kfp_ray/doc_id_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:latest" @@ -25,7 +30,10 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv( + "KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH +) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/ededup/kfp_ray/ededup_wf.py b/transforms/universal/ededup/kfp_ray/ededup_wf.py index 29adf5c18..17c85b630 100644 --- a/transforms/universal/ededup/kfp_ray/ededup_wf.py +++ b/transforms/universal/ededup/kfp_ray/ededup_wf.py @@ -15,7 +15,12 @@ import kfp.components as comp import kfp.dsl as dsl from src.ededup_compute_execution_params import ededup_compute_execution_params -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:latest" @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # KFPv1 and KFP2 uses different methods to create a component from a function. KFPv1 uses the # `create_component_from_func` function, but it is deprecated by KFPv2 and so has a different import path. diff --git a/transforms/universal/fdedup/kfp_ray/fdedup_wf.py b/transforms/universal/fdedup/kfp_ray/fdedup_wf.py index 6b1265cf8..51ead9c79 100644 --- a/transforms/universal/fdedup/kfp_ray/fdedup_wf.py +++ b/transforms/universal/fdedup/kfp_ray/fdedup_wf.py @@ -21,7 +21,7 @@ get_duplicate_list_compute_execution_params, signature_calc_compute_execution_params, ) -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, DEFAULT_KFP_COMPONENT_SPEC_PATH, ComponentUtils task_image = os.getenv("FDEDUP_IMAGE_LOCATION", "quay.io/dataprep1/data-prep-kit/fdedup-ray:latest") @@ -37,7 +37,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # KFPv1 and KFP2 uses different methods to create a component from a function. KFPv1 uses the # `create_component_from_func` function, but it is deprecated by KFPv2 and so has a different import path. diff --git a/transforms/universal/filter/kfp_ray/filter_wf.py b/transforms/universal/filter/kfp_ray/filter_wf.py index e40fff775..a18d2796d 100644 --- a/transforms/universal/filter/kfp_ray/filter_wf.py +++ b/transforms/universal/filter/kfp_ray/filter_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/hap/kfp_ray/hap_wf.py b/transforms/universal/hap/kfp_ray/hap_wf.py index 2008f091e..64c80fe37 100644 --- a/transforms/universal/hap/kfp_ray/hap_wf.py +++ b/transforms/universal/hap/kfp_ray/hap_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/hap-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/noop/kfp_ray/noop_multiple_wf.py b/transforms/universal/noop/kfp_ray/noop_multiple_wf.py index 3b102d205..9ed874f3d 100644 --- a/transforms/universal/noop/kfp_ray/noop_multiple_wf.py +++ b/transforms/universal/noop/kfp_ray/noop_multiple_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/noop-ray:latest" @@ -26,7 +31,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/noop/kfp_ray/noop_wf.py b/transforms/universal/noop/kfp_ray/noop_wf.py index e8125328b..5a1ce393a 100644 --- a/transforms/universal/noop/kfp_ray/noop_wf.py +++ b/transforms/universal/noop/kfp_ray/noop_wf.py @@ -15,7 +15,12 @@ import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/noop-ray:latest" @@ -27,7 +32,7 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/profiler/kfp_ray/profiler_wf.py b/transforms/universal/profiler/kfp_ray/profiler_wf.py index 914637895..7a157c146 100644 --- a/transforms/universal/profiler/kfp_ray/profiler_wf.py +++ b/transforms/universal/profiler/kfp_ray/profiler_wf.py @@ -15,7 +15,12 @@ import kfp.components as comp import kfp.dsl as dsl from src.profiler_compute_execution_params import profiler_compute_execution_params -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/profiler-ray:latest" @@ -27,7 +32,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # KFPv1 and KFP2 uses different methods to create a component from a function. KFPv1 uses the # `create_component_from_func` function, but it is deprecated by KFPv2 and so has a different import path. diff --git a/transforms/universal/resize/kfp_ray/resize_wf.py b/transforms/universal/resize/kfp_ray/resize_wf.py index 0724ed731..6a1403f18 100644 --- a/transforms/universal/resize/kfp_ray/resize_wf.py +++ b/transforms/universal/resize/kfp_ray/resize_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) task_image = "quay.io/dataprep1/data-prep-kit/resize-ray:latest" @@ -25,7 +30,8 @@ base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As # a result, instead of creating a component we are creating it in place here. diff --git a/transforms/universal/tokenization/kfp_ray/tokenization_wf.py b/transforms/universal/tokenization/kfp_ray/tokenization_wf.py index c9fb6f2e9..82fc55ae2 100644 --- a/transforms/universal/tokenization/kfp_ray/tokenization_wf.py +++ b/transforms/universal/tokenization/kfp_ray/tokenization_wf.py @@ -14,7 +14,12 @@ import kfp.compiler as compiler import kfp.components as comp import kfp.dsl as dsl -from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils +from workflow_support.compile_utils import ( + DEFAULT_KFP_COMPONENT_SPEC_PATH, + ONE_HOUR_SEC, + ONE_WEEK_SEC, + ComponentUtils, +) # the name of the job script @@ -27,7 +32,8 @@ # path to kfp component specifications files # path to kfp component specifications files -component_spec_path = "../../../../kfp/kfp_ray_components/" +component_spec_path = os.getenv("KFP_COMPONENT_SPEC_PATH", DEFAULT_KFP_COMPONENT_SPEC_PATH) + # compute execution parameters. Here different transforms might need different implementations. As