From 3486bca9c93267d2afa30736483b1fbf05b941ab Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 23 Jul 2024 12:59:08 -0400 Subject: [PATCH] Fix Dockerfiles to doc the copied main() src file. remove RUNTIME from Makefiles Signed-off-by: David Wood --- transforms/code/code2parquet/python/Dockerfile | 8 +++----- transforms/code/code2parquet/python/Makefile | 1 - transforms/code/code2parquet/ray/Dockerfile | 4 +++- transforms/code/code_quality/python/Dockerfile | 4 +++- transforms/code/code_quality/ray/Dockerfile | 4 +++- transforms/code/malware/ray/Dockerfile | 3 +++ transforms/code/proglang_select/python/Dockerfile | 5 +++-- transforms/code/proglang_select/ray/Dockerfile | 4 +++- transforms/language/lang_id/python/Dockerfile | 4 +++- transforms/language/lang_id/python/Makefile | 1 - transforms/language/lang_id/ray/Dockerfile | 4 +++- transforms/universal/doc_id/ray/Dockerfile | 4 +++- transforms/universal/doc_id/spark/Dockerfile | 4 +++- transforms/universal/ededup/ray/Dockerfile | 4 +++- transforms/universal/fdedup/ray/Dockerfile | 4 +++- transforms/universal/filter/python/Dockerfile | 8 +++----- transforms/universal/filter/ray/Dockerfile | 4 +++- transforms/universal/filter/spark/Dockerfile | 5 +++-- transforms/universal/noop/python/Dockerfile | 8 +++----- transforms/universal/noop/python/Makefile | 1 - transforms/universal/noop/ray/Dockerfile | 4 +++- transforms/universal/noop/spark/Dockerfile | 4 +++- transforms/universal/profiler/ray/Dockerfile | 4 +++- transforms/universal/resize/python/Dockerfile | 4 +++- transforms/universal/resize/python/Makefile | 1 - transforms/universal/resize/ray/Dockerfile | 3 +++ transforms/universal/tokenization/python/Dockerfile | 6 +++--- transforms/universal/tokenization/ray/Dockerfile | 4 +++- 28 files changed, 72 insertions(+), 42 deletions(-) diff --git a/transforms/code/code2parquet/python/Dockerfile b/transforms/code/code2parquet/python/Dockerfile index 0c75c342c..c0af4bd78 100644 --- a/transforms/code/code2parquet/python/Dockerfile +++ b/transforms/code/code2parquet/python/Dockerfile @@ -21,12 +21,10 @@ COPY --chown=dpk:root src/ src/ COPY --chown=dpk:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -#COPY requirements.txt requirements.txt -#RUN pip install --no-cache-dir -r requirements.txt - -# copy source data -COPY ./src/code2parquet_transform.py . +# copy the main() entry point to the image COPY ./src/code2parquet_transform_python.py . + +# copy some of the samples in COPY ./src/code2parquet_local.py local/ # copy test diff --git a/transforms/code/code2parquet/python/Makefile b/transforms/code/code2parquet/python/Makefile index b358e944d..6e4c9be0f 100644 --- a/transforms/code/code2parquet/python/Makefile +++ b/transforms/code/code2parquet/python/Makefile @@ -7,7 +7,6 @@ REPOROOT=../../../.. # $(REPOROOT)/.make.versions file contains the versions -TRANSFORM_RUNTIME=python TRANSFORM_NAME=code2parquet include $(REPOROOT)/transforms/.make.transforms diff --git a/transforms/code/code2parquet/ray/Dockerfile b/transforms/code/code2parquet/ray/Dockerfile index 9fe96df06..b679d241f 100644 --- a/transforms/code/code2parquet/ray/Dockerfile +++ b/transforms/code/code2parquet/ray/Dockerfile @@ -21,8 +21,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source +# copy the main() entry point to the image COPY src/code2parquet_transform_ray.py . + +# copy some of the samples in COPY src/code2parquet_local_ray.py local/ # copy test diff --git a/transforms/code/code_quality/python/Dockerfile b/transforms/code/code_quality/python/Dockerfile index 7117d1b16..a18a1c9d6 100644 --- a/transforms/code/code_quality/python/Dockerfile +++ b/transforms/code/code_quality/python/Dockerfile @@ -24,8 +24,10 @@ RUN pip install --no-cache-dir -e . #COPY requirements.txt requirements.txt #RUN pip install --no-cache-dir -r requirements.txt -# copy source data +# copy the main() entry point to the image COPY ./src/code_quality_transform_python.py . + +# copy some of the samples in COPY ./src/code_quality_local.py local/ # copy test diff --git a/transforms/code/code_quality/ray/Dockerfile b/transforms/code/code_quality/ray/Dockerfile index ae6bf3463..71af1c678 100644 --- a/transforms/code/code_quality/ray/Dockerfile +++ b/transforms/code/code_quality/ray/Dockerfile @@ -25,8 +25,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/code_quality_transform_ray.py . + +# copy some of the samples in COPY ./src/code_quality_local_ray.py local/ # copy test diff --git a/transforms/code/malware/ray/Dockerfile b/transforms/code/malware/ray/Dockerfile index 40f1815b4..589e06d7b 100644 --- a/transforms/code/malware/ray/Dockerfile +++ b/transforms/code/malware/ray/Dockerfile @@ -47,7 +47,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . +# copy the main() entry point to the image COPY src/malware_transform_ray.py ./ + +# copy some of the samples in COPY src/malware_local_ray.py local/ COPY test/ test/ diff --git a/transforms/code/proglang_select/python/Dockerfile b/transforms/code/proglang_select/python/Dockerfile index 8e7060746..fcff730a8 100644 --- a/transforms/code/proglang_select/python/Dockerfile +++ b/transforms/code/proglang_select/python/Dockerfile @@ -21,9 +21,10 @@ COPY --chown=dpk:root src/ src/ COPY --chown=dpk:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . - -# copy source data +# copy the main() entry point to the image COPY ./src/proglang_select_transform_python.py . + +# copy some of the samples in COPY ./src/proglang_select_local.py local/ # copy test diff --git a/transforms/code/proglang_select/ray/Dockerfile b/transforms/code/proglang_select/ray/Dockerfile index e37552125..1e5a07afb 100644 --- a/transforms/code/proglang_select/ray/Dockerfile +++ b/transforms/code/proglang_select/ray/Dockerfile @@ -21,8 +21,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/proglang_select_transform_ray.py . + +# copy some of the samples in COPY ./src/proglang_select_local_ray.py local/ # copy test diff --git a/transforms/language/lang_id/python/Dockerfile b/transforms/language/lang_id/python/Dockerfile index a67ac0fe6..e11e87615 100644 --- a/transforms/language/lang_id/python/Dockerfile +++ b/transforms/language/lang_id/python/Dockerfile @@ -37,8 +37,10 @@ USER dpk #COPY requirements.txt requirements.txt #RUN pip install --no-cache-dir -r requirements.txt -# copy source data +# copy the main() entry point to the image COPY ./src/lang_id_transform_python.py . + +# copy some of the samples in COPY ./src/lang_id_local.py local/ # copy test diff --git a/transforms/language/lang_id/python/Makefile b/transforms/language/lang_id/python/Makefile index fa55809df..8e8189c25 100644 --- a/transforms/language/lang_id/python/Makefile +++ b/transforms/language/lang_id/python/Makefile @@ -7,7 +7,6 @@ REPOROOT=../../../.. # $(REPOROOT)/.make.versions file contains the versions -TRANSFORM_RUNTIME=python TRANSFORM_NAME=lang_id include $(REPOROOT)/transforms/.make.transforms diff --git a/transforms/language/lang_id/ray/Dockerfile b/transforms/language/lang_id/ray/Dockerfile index 7f7fe999d..e24c2ebab 100644 --- a/transforms/language/lang_id/ray/Dockerfile +++ b/transforms/language/lang_id/ray/Dockerfile @@ -31,8 +31,10 @@ RUN sudo apt remove gcc g++ -y \ && sudo rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* USER ray -# copy source data +# copy the main() entry point to the image COPY ./src/lang_id_transform_ray.py . + +# copy some of the samples in COPY ./src/lang_id_local_ray.py local/ # copy test diff --git a/transforms/universal/doc_id/ray/Dockerfile b/transforms/universal/doc_id/ray/Dockerfile index fe8d4a8d4..8735a5c34 100644 --- a/transforms/universal/doc_id/ray/Dockerfile +++ b/transforms/universal/doc_id/ray/Dockerfile @@ -18,8 +18,10 @@ COPY --chown=ray:users pyproject.toml pyproject.toml COPY --chown=ray:users README.md README.md RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/doc_id_transform_ray.py . + +# copy some of the samples in COPY src/doc_id_local_ray.py local/ # copy test diff --git a/transforms/universal/doc_id/spark/Dockerfile b/transforms/universal/doc_id/spark/Dockerfile index 2fb10a200..22d098676 100644 --- a/transforms/universal/doc_id/spark/Dockerfile +++ b/transforms/universal/doc_id/spark/Dockerfile @@ -20,8 +20,10 @@ COPY --chown=spark:root src/ src/ COPY --chown=spark:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source main +# copy the main() entry point to the image COPY ./src/doc_id_transform_spark.py . + +# copy some of the samples in COPY ./src/doc_id_local.py local/ # copy test diff --git a/transforms/universal/ededup/ray/Dockerfile b/transforms/universal/ededup/ray/Dockerfile index 04138bb00..8c824f271 100644 --- a/transforms/universal/ededup/ray/Dockerfile +++ b/transforms/universal/ededup/ray/Dockerfile @@ -19,8 +19,10 @@ COPY --chown=ray:users README.md README.md COPY --chown=ray:users images/ images/ RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/ededup_transform_ray.py . + +# copy some of the samples in COPY src/ededup_local_ray.py local/ # copy test diff --git a/transforms/universal/fdedup/ray/Dockerfile b/transforms/universal/fdedup/ray/Dockerfile index bd828aff3..4b75110c8 100644 --- a/transforms/universal/fdedup/ray/Dockerfile +++ b/transforms/universal/fdedup/ray/Dockerfile @@ -19,8 +19,10 @@ COPY --chown=ray:users README.md README.md COPY --chown=ray:users images/ images/ RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/fdedup_transform_ray.py . + +# copy some of the samples in COPY src/fdedup_local_ray.py local/ # copy test diff --git a/transforms/universal/filter/python/Dockerfile b/transforms/universal/filter/python/Dockerfile index 5d8a00f7e..5830c032e 100644 --- a/transforms/universal/filter/python/Dockerfile +++ b/transforms/universal/filter/python/Dockerfile @@ -21,12 +21,10 @@ COPY --chown=dpk:root src/ src/ COPY --chown=dpk:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -#COPY requirements.txt requirements.txt -#RUN pip install --no-cache-dir -r requirements.txt - -# copy source data -COPY ./src/filter_transform.py . +# copy the main() entry point to the image COPY ./src/filter_transform_python.py . + +# copy some of the samples in COPY ./src/filter_local.py local/ # copy test diff --git a/transforms/universal/filter/ray/Dockerfile b/transforms/universal/filter/ray/Dockerfile index 6c5b8bf54..e62a5b2da 100644 --- a/transforms/universal/filter/ray/Dockerfile +++ b/transforms/universal/filter/ray/Dockerfile @@ -20,8 +20,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/filter_transform_ray.py . + +# copy some of the samples in COPY src/filter_local_ray.py local/ # copy test diff --git a/transforms/universal/filter/spark/Dockerfile b/transforms/universal/filter/spark/Dockerfile index 92c62cf1b..95a90561c 100644 --- a/transforms/universal/filter/spark/Dockerfile +++ b/transforms/universal/filter/spark/Dockerfile @@ -19,9 +19,10 @@ COPY --chown=spark:root src/ src/ COPY --chown=spark:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source main - +# copy the main() entry point to the image COPY ./src/filter_transform_spark.py . + +# copy some of the samples in COPY ./src/filter_local.py local/ # copy test diff --git a/transforms/universal/noop/python/Dockerfile b/transforms/universal/noop/python/Dockerfile index 13a0e42f5..d59e293bb 100644 --- a/transforms/universal/noop/python/Dockerfile +++ b/transforms/universal/noop/python/Dockerfile @@ -21,12 +21,10 @@ COPY --chown=dpk:root src/ src/ COPY --chown=dpk:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -#COPY requirements.txt requirements.txt -#RUN pip install --no-cache-dir -r requirements.txt - -# copy source data -COPY ./src/noop_transform.py . +# copy transform main() entry point to the image COPY ./src/noop_transform_python.py . + +# copy some of the samples in COPY ./src/noop_local.py local/ # copy test diff --git a/transforms/universal/noop/python/Makefile b/transforms/universal/noop/python/Makefile index 0406fbd0c..4f5258a01 100644 --- a/transforms/universal/noop/python/Makefile +++ b/transforms/universal/noop/python/Makefile @@ -7,7 +7,6 @@ REPOROOT=../../../.. # $(REPOROOT)/.make.versions file contains the versions -TRANSFORM_RUNTIME=python TRANSFORM_NAME=noop include $(REPOROOT)/transforms/.make.transforms diff --git a/transforms/universal/noop/ray/Dockerfile b/transforms/universal/noop/ray/Dockerfile index b5b2bd5bc..4ba0d0d43 100644 --- a/transforms/universal/noop/ray/Dockerfile +++ b/transforms/universal/noop/ray/Dockerfile @@ -20,8 +20,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/noop_transform_ray.py . + +# copy some of the samples in COPY ./src/noop_local_ray.py local/ # copy test diff --git a/transforms/universal/noop/spark/Dockerfile b/transforms/universal/noop/spark/Dockerfile index 3fb44635e..4aca0248b 100644 --- a/transforms/universal/noop/spark/Dockerfile +++ b/transforms/universal/noop/spark/Dockerfile @@ -20,8 +20,10 @@ COPY --chown=root:root src/ src/ COPY --chown=root:root pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy in the main() entry point to the image COPY ./src/noop_transform_spark.py . + +# Copy in some samples COPY ./src/noop_local_spark.py local/ # copy test diff --git a/transforms/universal/profiler/ray/Dockerfile b/transforms/universal/profiler/ray/Dockerfile index 750e795bf..21cebb313 100644 --- a/transforms/universal/profiler/ray/Dockerfile +++ b/transforms/universal/profiler/ray/Dockerfile @@ -18,8 +18,10 @@ COPY --chown=ray:users pyproject.toml pyproject.toml COPY --chown=ray:users README.md README.md RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY src/profiler_transform_ray.py . + +# copy some of the samples in COPY src/profiler_local_ray.py local/ # copy test diff --git a/transforms/universal/resize/python/Dockerfile b/transforms/universal/resize/python/Dockerfile index f13adb663..3ceb68cf3 100644 --- a/transforms/universal/resize/python/Dockerfile +++ b/transforms/universal/resize/python/Dockerfile @@ -21,8 +21,10 @@ COPY --chown=dpk:users pyproject.toml pyproject.toml COPY --chown=dpk:users README.md Readme.md RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/resize_transform_python.py . + +# copy some of the samples in COPY ./src/resize_local.py local/ # copy test diff --git a/transforms/universal/resize/python/Makefile b/transforms/universal/resize/python/Makefile index e3c1a09e3..f1a561bb2 100644 --- a/transforms/universal/resize/python/Makefile +++ b/transforms/universal/resize/python/Makefile @@ -7,7 +7,6 @@ REPOROOT=../../../.. # $(REPOROOT)/.make.versions file contains the versions -TRANSFORM_RUNTIME=python TRANSFORM_NAME=resize include $(REPOROOT)/transforms/.make.transforms diff --git a/transforms/universal/resize/ray/Dockerfile b/transforms/universal/resize/ray/Dockerfile index 8768b927b..75959a51d 100644 --- a/transforms/universal/resize/ray/Dockerfile +++ b/transforms/universal/resize/ray/Dockerfile @@ -17,7 +17,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . +# copy the main() entry point to the image COPY ./src/resize_transform_ray.py ./ + +# copy some of the samples in COPY ./src/resize_local_ray.py local/ # Install pytest so we can test the image later diff --git a/transforms/universal/tokenization/python/Dockerfile b/transforms/universal/tokenization/python/Dockerfile index f14664e1e..f50cbd743 100644 --- a/transforms/universal/tokenization/python/Dockerfile +++ b/transforms/universal/tokenization/python/Dockerfile @@ -24,10 +24,10 @@ RUN pip install --no-cache-dir -e . #COPY requirements.txt requirements.txt #RUN pip install --no-cache-dir -r requirements.txt -# copy source data -COPY ./src/tokenization_transform.py . +# copy the main() entry point to the image COPY ./src/tokenization_transform_python.py . -COPY ./src/tokenization_utils.py . + +# copy some of the samples in COPY src/tokenization_local_python.py local/ # copy test diff --git a/transforms/universal/tokenization/ray/Dockerfile b/transforms/universal/tokenization/ray/Dockerfile index 25a8e3973..3414d7b59 100644 --- a/transforms/universal/tokenization/ray/Dockerfile +++ b/transforms/universal/tokenization/ray/Dockerfile @@ -21,8 +21,10 @@ COPY --chown=ray:users src/ src/ COPY --chown=ray:users pyproject.toml pyproject.toml RUN pip install --no-cache-dir -e . -# copy source data +# copy the main() entry point to the image COPY ./src/tokenization_transform_ray.py . + +# copy some of the samples in COPY src/tokenization_local_ray.py local/ # copy test