From b5b879337a148a118c86a7028b1056b7685fb6b9 Mon Sep 17 00:00:00 2001 From: Carlos Vivar Rios Date: Thu, 7 Nov 2024 12:30:04 +0100 Subject: [PATCH] feat: Password Protected Links --- .env.dist | 2 + .../workflows/multiplatform_docker_build.yml | 57 +++++++++++++++++++ .../multiplatform_docker_build_dockerhub.yaml | 56 ++++++++++++++++++ .gitignore | 7 +++ Dockerfile | 8 +++ README.md | 30 +++++++++- app/app.sh | 24 +++++++- odtp.yml | 11 +++- 8 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 .env.dist create mode 100644 .github/workflows/multiplatform_docker_build.yml create mode 100644 .github/workflows/multiplatform_docker_build_dockerhub.yaml diff --git a/.env.dist b/.env.dist new file mode 100644 index 0000000..e328f35 --- /dev/null +++ b/.env.dist @@ -0,0 +1,2 @@ +LINK= +PASSWORD= \ No newline at end of file diff --git a/.github/workflows/multiplatform_docker_build.yml b/.github/workflows/multiplatform_docker_build.yml new file mode 100644 index 0000000..3e4736b --- /dev/null +++ b/.github/workflows/multiplatform_docker_build.yml @@ -0,0 +1,57 @@ +name: Multi-Platform Docker Build + +on: + workflow_dispatch: + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + # Step 1: Check out the repository and submodules + - name: Check out code + uses: actions/checkout@v3 + with: + submodules: true # Fetch submodules + fetch-depth: 0 # Ensure the full history is fetched + + # Step 2: Set up Docker Buildx + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Step 3: Install yq + - name: Install yq + run: | + sudo apt-get update && sudo apt-get install -y wget + sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + # Step 4: Extract component-version and component-name from odtp.yml + - name: Extract component-version and component-name + id: extract_info + run: | + VERSION=$(yq e '.component-version' odtp.yml) + NAME=$(yq e '.component-name' odtp.yml) + echo "VERSION=${VERSION}" + echo "NAME=${NAME}" + echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV + echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV + + # Step 5: Log in to GitHub Container Registry + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Step 6: Build and push Docker image for multiple platforms + - name: Build and push Docker image + run: | + IMAGE_NAME=ghcr.io/${{ github.repository }}/${{ env.COMPONENT_NAME }} + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \ + -t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \ + -t $IMAGE_NAME:latest \ + --push . diff --git a/.github/workflows/multiplatform_docker_build_dockerhub.yaml b/.github/workflows/multiplatform_docker_build_dockerhub.yaml new file mode 100644 index 0000000..2bad773 --- /dev/null +++ b/.github/workflows/multiplatform_docker_build_dockerhub.yaml @@ -0,0 +1,56 @@ +name: Multi-Platform Docker Build for Dockerhub + +on: + workflow_dispatch: + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + # Step 1: Check out the repository and submodules + - name: Check out code + uses: actions/checkout@v3 + with: + submodules: true # Fetch submodules + fetch-depth: 0 # Ensure the full history is fetched + + # Step 2: Set up Docker Buildx + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Step 3: Install yq + - name: Install yq + run: | + sudo apt-get update && sudo apt-get install -y wget + sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + # Step 4: Extract component-version and component-name from odtp.yml + - name: Extract component-version and component-name + id: extract_info + run: | + VERSION=$(yq e '.component-version' odtp.yml) + NAME=$(yq e '.component-name' odtp.yml) + echo "VERSION=${VERSION}" + echo "NAME=${NAME}" + echo "COMPONENT_VERSION=${VERSION}" >> $GITHUB_ENV + echo "COMPONENT_NAME=${NAME}" >> $GITHUB_ENV + + # Step 5: Log in to Docker Hub + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Step 6: Build and push Docker image for multiple platforms + - name: Build and push Docker image + run: | + IMAGE_NAME=${{ secrets.DOCKER_USERNAME }}/${{ env.COMPONENT_NAME }} + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg COMPONENT_VERSION=${{ env.COMPONENT_VERSION }} \ + -t $IMAGE_NAME:${{ env.COMPONENT_VERSION }} \ + -t $IMAGE_NAME:latest \ + --push . diff --git a/.gitignore b/.gitignore index b6e4761..77b3ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# ODTP Testing +odtp-input/ +odtp-output/ + +# Mac crap +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index 3f661cf..50224dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,4 +66,12 @@ COPY ./odtp-component-client /odtp/odtp-component-client COPY ./app /odtp/odtp-app WORKDIR /odtp +# Avoid error when building on windows +RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/log.sh +RUN sed -i 's/\r$//' /odtp/odtp-component-client/src/shell/traceback.sh +RUN sed -i 's/\r$//' /odtp/odtp-component-client/scripts/component-update.sh +RUN sed -i 's/\r$//' /odtp/odtp-component-client/odtp-app.sh +RUN sed -i 's/\r$//' /odtp/odtp-component-client/startup.sh +RUN sed -i 's/\r$//' /odtp/odtp-app/app.sh + ENTRYPOINT ["bash", "/odtp/odtp-component-client/startup.sh"] \ No newline at end of file diff --git a/README.md b/README.md index af34047..a65ee59 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is the component that will download and arrange files to make it compatible ``` odtp new odtp-component-entry \ --name odtp-eqasim-dataloader \ ---component-version 0.3.2 \ +--component-version 0.4.0 \ --repository https://github.com/odtp-org/odtp-eqasim-dataloader ``` @@ -14,12 +14,36 @@ odtp new odtp-component-entry \ This component is a temporal solution until we have dataloaders defined for each source of data. This component accept one link to switchdrive and the download and uncompress it content to an output folder. -Therefore, the only ENV variable available is: +- LINK: Link to the switch drive. +- PASSWORD: Optional: If the link to the switch drive contains a password it should be provided. + +## Tutorial + +### How to run this component as docker for development + +Build the dockerfile + +``` +docker build -t odtp-eqasim-dataloader . +``` + +Run the following command. Mount the correct volumes for input/output folders. + +``` +docker run -it --rm -v ${pwd}/odtp-input:/odtp/odtp-input -v ${pwd}/odtp-output:/odtp/odtp-output --env-file .env --entrypoint bash odtp-eqasim-dataloader +``` -- LINK: ## Changelog +- v0.4.0 + +- v0.3.5 + +- v0.3.4 + +- v0.3.3 + - v0.3.2 - Ubuntu fixed at 22.04 - Python fixed at 3.10 diff --git a/app/app.sh b/app/app.sh index 4acc104..8fc4517 100644 --- a/app/app.sh +++ b/app/app.sh @@ -3,6 +3,26 @@ # Downloading Link provided # The option --progress=bar:force:noscroll makes the log output less verbose # While the progressbar is displayed only one line is added for it in the logs -wget --progress=bar:force:noscroll $LINK/download -mv download download.zip +wget --progress=bar:force:noscroll -O download.zip $LINK/download +echo "Downloaded file with user id and password. Now procedding with unzip" unzip -q download.zip -d /odtp/odtp-output +echo "Uncompressed file." + +if [ -n "$PASSWORD" ]; then + # Check if the required environment variables are set + if [ -z "$LINK" ] || [ -z "$PASSWORD" ]; then + echo "Error: LINK and PASSWORD environment variables must be set." + exit 1 + fi + + # Extract the user id from the LINK + USER_ID=$(echo $LINK | sed 's#.*/s/\(.*\)#\1#') + + # Download the file using wget with the provided user id and password + wget --user=$USER_ID --password=$PASSWORD --progress=bar:force:noscroll --header='X-Requested-With: XMLHttpRequest' -O download.zip https://drive.switch.ch/public.php/webdav/ + + echo "Downloaded file with user id and password. Now procedding with unzip" + unzip -qo download.zip -d /odtp/odtp-output + + echo "Uncompressed file." +fi diff --git a/odtp.yml b/odtp.yml index 9e0e8f0..4b256b0 100644 --- a/odtp.yml +++ b/odtp.yml @@ -1,7 +1,7 @@ # This file should contain basic component information for your component. component-name: odtp-eqasim-dataloader component-author: Carlos Vivar Rios -component-version: 0.3.1 +component-version: v0.4.0 component-repository: https://github.com/odtp-org/odtp-eqasim-dataloader component-license: AGPL-3.0 component-type: ephemeral @@ -15,7 +15,13 @@ tools: null # If your tool require some secrets token to be passed as ENV to the component # This won't be traced in MongoDB -secrets: null +secrets: + - name: LINK + default-value: null + datatype: string + description: Password to the switch drive link to download + options: null + allow-custom-value: true # If the tool requires some building arguments such as Matlab license build-args: null @@ -34,6 +40,7 @@ parameters: options: null allow-custom-value: true + # If applicable, data-input list required by the component data-inputs: null