From fdee6a7077e59112dc97e8ddded255722146fefe Mon Sep 17 00:00:00 2001 From: Maroun Touma Date: Wed, 23 Oct 2024 14:45:16 -0500 Subject: [PATCH] addional fixes for spark stuff Signed-off-by: Maroun Touma --- .make.defaults | 2 +- tools/ingest2parquet/Dockerfile | 7 +++---- tools/ingest2parquet/Makefile | 2 +- transforms/code/code_profiler/python/requirements.txt | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.make.defaults b/.make.defaults index 8e8cc981e..0447f2463 100644 --- a/.make.defaults +++ b/.make.defaults @@ -449,7 +449,7 @@ else endif if [ -d ../python ]; then \ source venv/bin/activate; \ - $(MAKE) PYTHON_PROJECT_BUILD_DIR=.. .defaults.install-src-venv; \ + $(MAKE) PYTHON_PROJECT_BUILD_DIR=../python .defaults.install-src-venv; \ fi # Run tests in test directory from that dir after adding ../src to PYTHONPATH diff --git a/tools/ingest2parquet/Dockerfile b/tools/ingest2parquet/Dockerfile index 4aaed5f9c..a12ba7fa9 100644 --- a/tools/ingest2parquet/Dockerfile +++ b/tools/ingest2parquet/Dockerfile @@ -4,13 +4,12 @@ FROM ${BASE_IMAGE} # install pytest RUN pip install --no-cache-dir pytest +ARG WHEEL_FILE_NAME # Copy and install data processing libraries # These are expected to be placed in the docker context before this is run (see the make image). -COPY --chown=ray:users data-processing-lib-python/ data-processing-lib-python/ -RUN cd data-processing-lib-python && pip install --no-cache-dir -e . -COPY --chown=ray:users data-processing-lib-ray/ data-processing-lib-ray/ -RUN cd data-processing-lib-ray && pip install --no-cache-dir -e . +COPY --chown=ray:users data-processing-dist data-processing-dist +RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray] COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt diff --git a/tools/ingest2parquet/Makefile b/tools/ingest2parquet/Makefile index c932000a6..47f5a0a31 100644 --- a/tools/ingest2parquet/Makefile +++ b/tools/ingest2parquet/Makefile @@ -16,7 +16,7 @@ test:: venv test-src test-image clean:: .defaults.clean -image:: .defaults.ray-lib-src-image +image:: .defaults.ray-lib-whl-image test-src:: .defaults.test-src test-local diff --git a/transforms/code/code_profiler/python/requirements.txt b/transforms/code/code_profiler/python/requirements.txt index 5a90edfaa..9946becee 100644 --- a/transforms/code/code_profiler/python/requirements.txt +++ b/transforms/code/code_profiler/python/requirements.txt @@ -1,4 +1,4 @@ -data-prep-toolkit==0.2.2.dev0 +data-prep-toolkit==0.2.2.dev1 parameterized pandas aiolimiter==1.1.0