From eb5f9894163f8bbcf88b0a98aeefbabaf279b231 Mon Sep 17 00:00:00 2001 From: Maroun Touma Date: Wed, 18 Dec 2024 07:07:02 -0500 Subject: [PATCH] remove remote image from .make.defaults Signed-off-by: Maroun Touma --- .make.defaults | 3 +-- kfp/kfp_ray_components/Makefile | 3 +++ transforms/.make.cicd.targets | 3 ++- transforms/.make.transforms | 4 ++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.make.defaults b/.make.defaults index 4674326d8..8bb53e487 100644 --- a/.make.defaults +++ b/.make.defaults @@ -47,7 +47,6 @@ DOCKER_REGISTRY_USER?=$(DPK_DOCKER_REGISTRY_USER) DOCKER_REGISTRY_KEY?=$(DPK_DOCKER_REGISTRY_KEY) DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE) DOCKER_LOCAL_IMAGE=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) -DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-$(SPARK_VERSION) DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) RAY_BASE_IMAGE?=docker.io/rayproject/ray:${RAY}-py310 @@ -231,7 +230,7 @@ __check_defined = \ --build-arg GIT_COMMIT=$(shell git log -1 --format=%h) . $(DOCKER) tag $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) $(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) - + # Copy a source tree in LIB_PATH, including src, pyproject.toml to LIB_NAME # Generally used to copy source from within the repo into a local directory for use by a Dockerfile .PHONY: .defaults.copy-lib diff --git a/kfp/kfp_ray_components/Makefile b/kfp/kfp_ray_components/Makefile index 17488bc2e..cac6047ce 100644 --- a/kfp/kfp_ray_components/Makefile +++ b/kfp/kfp_ray_components/Makefile @@ -16,6 +16,9 @@ else DOCKER_IMAGE_NAME=kfp-data-processing endif +## removed from .make.defaults and set here before loading in kind +DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) + .PHONY: .lib-src-image .lib-src-image:: .default.build-lib-wheel @$(eval LIB_WHEEL_FILE := $(shell find data-processing-dist/*.whl)) diff --git a/transforms/.make.cicd.targets b/transforms/.make.cicd.targets index 95f310a4e..9d2780849 100644 --- a/transforms/.make.cicd.targets +++ b/transforms/.make.cicd.targets @@ -17,7 +17,6 @@ TRANSFORM_RAY_SRC?="-m dpk_$(TRANSFORM_NAME).ray.transform" TRANSFORM_SPARK_SRC?="-m dpk_$(TRANSFORM_NAME).spark.transform" - venv:: .defaults.create-venv source venv/bin/activate && $(PIP) install -e $(REPOROOT)/data-processing-lib[ray,spark] source venv/bin/activate && $(PIP) install -e $(REPOROOT)/data-connector-lib @@ -67,6 +66,7 @@ test-image:: .default.build-lib-wheel $(MAKE) DOCKER_FILE=Dockerfile.python \ TRANSFORM_RUNTIME_SRC_FILE=$(TRANSFORM_PYTHON_SRC) \ DOCKER_IMAGE_NAME=$(TRANSFORM_NAME)-python \ + DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \ test-image-sequence ; \ fi ;\ fi @@ -76,6 +76,7 @@ test-image:: .default.build-lib-wheel TRANSFORM_RUNTIME_SRC_FILE=$(TRANSFORM_RAY_SRC) \ DOCKER_IMAGE_NAME=$(TRANSFORM_NAME)-ray \ BASE_IMAGE=$(RAY_BASE_IMAGE) \ + DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) \ test-image-sequence ; \ fi ;\ fi diff --git a/transforms/.make.transforms b/transforms/.make.transforms index 247ea2ae4..860d08a46 100644 --- a/transforms/.make.transforms +++ b/transforms/.make.transforms @@ -35,6 +35,10 @@ DOCKER_IMAGE_NAME?=$(TRANSFORM_NAME)-$(TRANSFORM_RUNTIME) TRANSFORM_RUNTIME_SRC_FILE?=$(TRANSFORM_NAME)_transform_$(TRANSFORM_RUNTIME).py TRANSFORM_TEST_FILE?=test/test_$(TRANSFORM_NAME).py +## Used when publish or building the image by legacy tansforms. +## Will be removed when all transfdorms get refactored +DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) + # The following taken from https://stackoverflow.com/a/65243296/45375 # Lists all targets and optional help text found in the target. # :: rule means we first run the help from the main make.defaults.