addional fixes for spark stuff

Signed-off-by: Maroun Touma <[email protected]>

.make.defaults

@@ -449,7 +449,7 @@ else
 endif
 	if [ -d ../python ]; then                                               	\
 		source venv/bin/activate;				      		\
-		$(MAKE) PYTHON_PROJECT_BUILD_DIR=.. .defaults.install-src-venv;    	\
+		$(MAKE) PYTHON_PROJECT_BUILD_DIR=../python .defaults.install-src-venv;    	\
 	fi
 
 # Run tests in test directory from that dir after adding ../src to PYTHONPATH

tools/ingest2parquet/Dockerfile

@@ -4,13 +4,12 @@ FROM ${BASE_IMAGE}
 
 # install pytest
 RUN pip install --no-cache-dir pytest
+ARG WHEEL_FILE_NAME
 
 # Copy and install data processing libraries 
 # These are expected to be placed in the docker context before this is run (see the make image).
-COPY --chown=ray:users data-processing-lib-python/ data-processing-lib-python/
-RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
-COPY --chown=ray:users data-processing-lib-ray/ data-processing-lib-ray/ 
-RUN cd data-processing-lib-ray    && pip install --no-cache-dir -e .
+COPY --chown=ray:users data-processing-dist data-processing-dist
+RUN  pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
 
 COPY requirements.txt requirements.txt
 RUN pip install --no-cache-dir -r  requirements.txt

tools/ingest2parquet/Makefile

@@ -16,7 +16,7 @@ test::	venv test-src test-image
 
 clean:: .defaults.clean
 
-image:: .defaults.ray-lib-src-image
+image:: .defaults.ray-lib-whl-image
 
 test-src:: .defaults.test-src test-local
 

transforms/code/code_profiler/python/requirements.txt

@@ -1,4 +1,4 @@
-data-prep-toolkit==0.2.2.dev0
+data-prep-toolkit==0.2.2.dev1
 parameterized
 pandas
 aiolimiter==1.1.0