html2parquet Moved to language folder

.github/workflows/test-universal-html2parquet.yml → .github/workflows/test-language-html2parquet.yml

@@ -1,7 +1,7 @@
 #
 # DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
-name: Test - transforms/universal/html2parquet
+name: Test - transforms/language/html2parquet
 
 on:
     workflow_dispatch:
@@ -12,9 +12,9 @@ on:
         tags:
             - "*"
         paths:
-            - "transforms/universal/html2parquet/**"
+            - "transforms/language/html2parquet/**"
             - "data-processing-lib/**"
-            - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
+            - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
             - "!data-processing-lib/**/test/**"
             - "!data-processing-lib/**/test-data/**"
             - "!**.md"
@@ -26,9 +26,9 @@ on:
             - "dev"
             - "releases/**"
         paths:
-            - "transforms/universal/html2parquet/**"
+            - "transforms/language/html2parquet/**"
             - "data-processing-lib/**"
-            - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
+            - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
             - "!data-processing-lib/**/test/**"
             - "!data-processing-lib/**/test-data/**"
             - "!**.md"
@@ -72,12 +72,12 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test transform source in transforms/universal/html2parquet
+            - name: Test transform source in transforms/language/html2parquet
               run: |
-                  if [ -e "transforms/universal/html2parquet/Makefile" ]; then
-                      make -C transforms/universal/html2parquet DOCKER=docker test-src
+                  if [ -e "transforms/language/html2parquet/Makefile" ]; then
+                      make -C transforms/language/html2parquet DOCKER=docker test-src
                   else
-                      echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform."
+                      echo "transforms/language/html2parquet/Makefile not found - source testing disabled for this transform."
                   fi
     test-image:
         needs: [check_if_push_image]
@@ -99,15 +99,15 @@ jobs:
                   sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                   sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                   df -h
-            - name: Test transform image in transforms/universal/html2parquet
+            - name: Test transform image in transforms/language/html2parquet
               run: |
-                  if [ -e "transforms/universal/html2parquet/Makefile" ]; then
-                      if [ -d "transforms/universal/html2parquet/spark" ]; then
+                  if [ -e "transforms/language/html2parquet/Makefile" ]; then
+                      if [ -d "transforms/language/html2parquet/spark" ]; then
                           make -C data-processing-lib/spark DOCKER=docker image
                       fi
-                      make -C transforms/universal/html2parquet DOCKER=docker test-image
+                      make -C transforms/language/html2parquet DOCKER=docker test-image
                   else
-                      echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform."
+                      echo "transforms/language/html2parquet/Makefile not found - testing disabled for this transform."
                   fi
             - name: Print space
               # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -117,8 +117,8 @@ jobs:
             - name: Publish images
               if: needs.check_if_push_image.outputs.publish_images == 'true'
               run: |
-                  if [ -e "transforms/universal/html2parquet/Makefile" ]; then
-                      make -C transforms/universal/html2parquet publish
+                  if [ -e "transforms/language/html2parquet/Makefile" ]; then
+                      make -C transforms/language/html2parquet publish
                   else
-                      echo "transforms/universal/html2parquet/Makefile not found - publishing disabled for this transform."
+                      echo "transforms/language/html2parquet/Makefile not found - publishing disabled for this transform."
                   fi

transforms/universal/html2parquet/python/Dockerfile → transforms/language/html2parquet/python/Dockerfile

@@ -19,6 +19,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy transform main() entry point to the image 

transforms/universal/html2parquet/python/pyproject.toml → transforms/language/html2parquet/python/pyproject.toml

@@ -9,16 +9,15 @@ authors = [
     { name = "Sungeun An", email = "[email protected]" },
     { name = "Syed Zawad", email = "[email protected]" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-    "trafilatura==1.12.0"
-
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/language/html2parquet/python/requirements.txt

@@ -0,0 +1,2 @@
+data-prep-toolkit==0.2.2.dev0
+trafilatura==1.12.0

transforms/universal/html2parquet/python/test/test_html2parquet_python.py → transforms/language/html2parquet/python/test/test_html2parquet_python.py

@@ -33,7 +33,6 @@ def get_test_transform_fixtures(self) -> list[tuple]:
             "html2parquet_output_format": "markdown",
         }
         # this is added as a fixture to remove these columns from comparison
-        ignore_columns = ["date_acquired", "document_id", "pdf_convert_time", "hash"]
         ignore_columns = ["date_acquired"]
 
         fixtures = []