feat: Password Protected Links

.env.dist

@@ -0,0 +1,2 @@
+LINK=
+PASSWORD=

.github/workflows/multiplatform_docker_build.yml

@@ -0,0 +1,57 @@
+name: Multi-Platform Docker Build
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Check out the repository and submodules
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: true  # Fetch submodules
+          fetch-depth: 0    # Ensure the full history is fetched
+
+      # Step 2: Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Step 3: Install yq
+      - name: Install yq
+        run: |
+          sudo apt-get update && sudo apt-get install -y wget
+          sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+
+      # Step 4: Extract component-version and component-name from odtp.yml
+      - name: Extract component-version and component-name
+        id: extract_info
+        run: |
+          VERSION=$(yq e '.component-version' odtp.yml)
+          NAME=$(yq e '.component-name' odtp.yml)
+          echo "VERSION=${VERSION}"
+          echo "NAME=${NAME}"
+          echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV
+
+      # Step 5: Log in to GitHub Container Registry
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Step 6: Build and push Docker image for multiple platforms
+      - name: Build and push Docker image
+        run: |
+          IMAGE_NAME=ghcr.io/${{ github.repository }}/${{ env.COMPONENT_NAME }}
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \
+            -t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \
+            -t $IMAGE_NAME:latest \
+            --push .

.github/workflows/multiplatform_docker_build_dockerhub.yaml

@@ -0,0 +1,56 @@
+name: Multi-Platform Docker Build for Dockerhub
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Check out the repository and submodules
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: true  # Fetch submodules
+          fetch-depth: 0    # Ensure the full history is fetched
+
+      # Step 2: Set up Docker Buildx
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Step 3: Install yq
+      - name: Install yq
+        run: |
+          sudo apt-get update && sudo apt-get install -y wget
+          sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+
+      # Step 4: Extract component-version and component-name from odtp.yml
+      - name: Extract component-version and component-name
+        id: extract_info
+        run: |
+          VERSION=$(yq e '.component-version' odtp.yml)
+          NAME=$(yq e '.component-name' odtp.yml)
+          echo "VERSION=${VERSION}"
+          echo "NAME=${NAME}"
+          echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV
+
+      # Step 5: Log in to Docker Hub
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Step 6: Build and push Docker image for multiple platforms
+      - name: Build and push Docker image
+        run: |
+          IMAGE_NAME=${{ secrets.DOCKER_USERNAME }}/${{ env.COMPONENT_NAME }}
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \
+            -t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \
+            -t $IMAGE_NAME:latest \
+            --push .

.gitignore

@@ -1,3 +1,10 @@
+# ODTP Testing
+odtp-input/
+odtp-output/
+
+# Mac crap
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Dockerfile

@@ -66,4 +66,12 @@ COPY ./odtp-component-client /odtp/odtp-component-client
 COPY ./app /odtp/odtp-app
 WORKDIR /odtp
 
+# Avoid error when building on windows
+RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/log.sh
+RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/traceback.sh
+RUN sed -i 's/\r$//' /odtp/odtp-component-client/scripts/component-update.sh
+RUN sed -i 's/\r$//' /odtp/odtp-component-client/odtp-app.sh
+RUN sed -i 's/\r$//' /odtp/odtp-component-client/startup.sh
+RUN sed -i 's/\r$//' /odtp/odtp-app/app.sh
+
 ENTRYPOINT ["bash", "/odtp/odtp-component-client/startup.sh"]

README.md

@@ -6,20 +6,44 @@ This is the component that will download and arrange files to make it compatible
 ```
 odtp new odtp-component-entry \
 --name odtp-eqasim-dataloader \
---component-version 0.3.2 \
+--component-version 0.4.0 \
 --repository https://github.com/odtp-org/odtp-eqasim-dataloader
 ```
 
 ## Tutorial
 
 This component is a temporal solution until we have dataloaders defined for each source of data. This component accept one link to switchdrive and the download and uncompress it content to an output folder. 
 
-Therefore, the only ENV variable available is: 
+- LINK: Link to the switch drive. 
+- PASSWORD: Optional: If the link to the switch drive contains a password it should be provided.
+
+## Tutorial
+
+### How to run this component as docker for development
+
+Build the dockerfile 
+
+```
+docker build -t odtp-eqasim-dataloader .
+```
+
+Run the following command. Mount the correct volumes for input/output folders. 
+
+```
+docker run -it --rm -v ${pwd}/odtp-input:/odtp/odtp-input -v ${pwd}/odtp-output:/odtp/odtp-output --env-file .env --entrypoint bash odtp-eqasim-dataloader
+```
 
-- LINK:
 
 ## Changelog
 
+- v0.4.0
+
+- v0.3.5
+
+- v0.3.4
+
+- v0.3.3
+
 - v0.3.2
     - Ubuntu fixed at 22.04
     - Python fixed at 3.10

app/app.sh

@@ -3,6 +3,26 @@
 # Downloading Link provided
 # The option --progress=bar:force:noscroll makes the log output less verbose
 # While the progressbar is displayed only one line is added for it in the logs
-wget --progress=bar:force:noscroll $LINK/download
-mv download download.zip
+wget --progress=bar:force:noscroll -O download.zip  $LINK/download
+echo "Downloaded file with user id and password. Now procedding with unzip"
 unzip -q download.zip -d /odtp/odtp-output
+echo "Uncompressed file."
+
+if [ -n "$PASSWORD" ]; then
+    # Check if the required environment variables are set
+    if [ -z "$LINK" ] || [ -z "$PASSWORD" ]; then
+        echo "Error: LINK and PASSWORD environment variables must be set."
+        exit 1
+    fi
+
+    # Extract the user id from the LINK
+    USER_ID=$(echo $LINK | sed 's#.*/s/\(.*\)#\1#')
+
+    # Download the file using wget with the provided user id and password
+    wget --user=$USER_ID --password=$PASSWORD --progress=bar:force:noscroll --header='X-Requested-With: XMLHttpRequest' -O download.zip https://drive.switch.ch/public.php/webdav/
+
+    echo "Downloaded file with user id and password. Now procedding with unzip"
+    unzip -qo download.zip -d /odtp/odtp-output
+
+    echo "Uncompressed file."
+fi

odtp.yml

@@ -1,7 +1,7 @@
 # This file should contain basic component information for your component.
 component-name: odtp-eqasim-dataloader
 component-author: Carlos Vivar Rios
-component-version: 0.3.1
+component-version: v0.4.0
 component-repository: https://github.com/odtp-org/odtp-eqasim-dataloader
 component-license: AGPL-3.0
 component-type: ephemeral
@@ -15,7 +15,13 @@ tools: null
 
 # If your tool require some secrets token to be passed as ENV to the component
 # This won't be traced in MongoDB
-secrets: null
+secrets: 
+  - name: LINK
+    default-value: null
+    datatype: string
+    description: Password to the switch drive link to download
+    options: null
+    allow-custom-value: true
 
 # If the tool requires some building arguments such as Matlab license
 build-args: null
@@ -34,6 +40,7 @@ parameters:
     options: null
     allow-custom-value: true
 
+
 # If applicable, data-input list required by the component
 data-inputs: null