chebi.yml

name: chebi

on:
  workflow_dispatch:
  pull_request: # tests whether it is working on PR
     paths:
     - '.`/workflows/chebi.yml'
  schedule:
    - cron: "0 0 1,15 * *"  # Run the workflow on the 1st and 15th day of each month
permissions:
  contents: write
  pages: write
  id-token: write
jobs:
  check_release:
    runs-on: ubuntu-latest
    name: Check latest release date
    steps:
      # checkout the repository
      - name: Checkout
        uses: actions/checkout@v4
      # Download current version from Zenodo
      - name: Download file from Zenodo
        env:
          zenodo_token: ${{ secrets.ZENODO }}
        run: |
          mkdir artifact_check
          cd artifact_check
          mkdir check_release_vars
          cd ../
          # Set up variables
          chmod +x datasources/chebi/config
          datasources/chebi/config .
          zenodo_file_id="8348142"
          file_name=$to_check_from_zenodo
          echo file name: $file_name
          echo 'Dowloading from: https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'
          mkdir check_release_vars
          #echo "FILE_NAME=$file_name" >> artifact_check/check_release_vars/config
          
          # Request Zenodo API to download the file
          curl -H "Authorization: Bearer $zenodo_token" -LJO 'https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'          
          # Verify the downloaded file
          ls -l $file_name   
      # check the release date for the latest ChEBI release
      - name: Check for new ChEBI release
        id: check_download
        run: |
          echo 'Accessing the ChEBI archive'
          wget https://ftp.ebi.ac.uk/pub/databases/chebi/archive/
          ## Read config
          . datasources/chebi/config .
          ## Check date for last element in index
          string=$(tail -4 index.html | head -1)
          ##Extract the date from the latest release (up to the day)
          date_new=$(echo "$string" | grep -oP '<td align="right">\K[0-9-]+\s[0-9:]+(?=\s+</td>)' | awk '{print $1}')
          release=$(echo "$string" | grep -oP '(?<=a href="rel)\d\d\d')
          echo "RELEASE=$release" >> artifact_check/check_release_vars/config
          echo "DATE_NEW=$date_new" >> artifact_check/check_release_vars/config
          ##Extract the date from the ChEBI README file 
          date_old=$date
          ##Compare the dates and set variable if date_new is more recent
          timestamp1=$(date -d "$date_new" +%s)
          timestamp2=$(date -d "$date_old" +%s)

          if [ "$timestamp1" -gt "$timestamp2" ]; then
            echo "DOWNLOAD_FILE=true" >> artifact_check/check_release_vars/config
            echo 'New release available', "$release"
          else
            echo "DOWNLOAD_FILE=false" >> artifact_check/check_release_vars/config
            echo 'No new release available'
          fi
          echo "Date of latest release: $date_new", "Date of release of the current version: $date_old"
          ##Clean up
          rm index.htm*
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: check_release_vars
          path: artifact_check/*

  process_file:
    name: Download and test latest ChEBI release
    needs: check_release
    runs-on: ubuntu-latest
    steps:
      - name: Download check_release_vars artifact
        uses: actions/download-artifact@v4
        with:
          name: check_release_vars
        env:
          zenodo_token: ${{ secrets.ZENODO }}
      - name: Download data
        run: |
          mkdir artifact_process
          cd artifact_process
          mkdir process_file
          cd ../
          ## Retrieve vars
          chmod +x check_release_vars/config
          check_release_vars/config .
          ##Download ChEBI SDF file
          wget https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE/SDF/ChEBI_complete_3star.sdf.gz
          ##Unzip gz file:
          gunzip ChEBI_complete_3star.sdf.gz

          ##Check file size if available
          ls
          ##Print file size
          FILENAME=$to_check_from_zenodo
          FILESIZE=$(stat -c%s "$FILENAME")
          echo "Size of $FILENAME = $FILESIZE bytes."

          ##Create temp. folder to store the data in
          mkdir -p mapping_preprocessing/datasources/chebi/data
          ##Move the SDF file to the expected location
          mv ChEBI_complete_3star.sdf mapping_preprocessing/datasources/chebi/data
          
     # step 3: run the R script for ChEBI preprocessing (to be made into a separate job)
      - name: Run R script to process SDF data
        run: |
          ##Check R version:
          R --version
          ##Install the readr package:
          #Rscript -e 'install.packages("readr")'
          ##Running the R script:
          #Rscript r/chebi_processing.R
          #echo "R preprocessing of ChEBI data is done"
     # step 4: run the Java .jar for ChEBI preprocessing (to be made into a separate job)
      - name: Set up Java
        uses: actions/setup-java@v2
        with:
          java-version: '11'
          distribution: 'adopt'

      - name: Test SDF processing
        run: |
          inputFile="mapping_preprocessing/datasources/chebi/data/ChEBI_complete_3star.sdf"
          outputDir="mapping_preprocessing/datasources/chebi/data"
          # Run Java program and capture its exit code
          java -cp target/mapping_prerocessing-0.0.1-jar-with-dependencies.jar org.sec2pri.chebi_sdf "$inputFile" "$outputDir"

          # Check the exit status of the Java program
          if [ $? -eq 0 ]; then
              # Java program succeeded
              echo "Java preprocessing of ChEBI data is done."
              echo "FAILED=false" >> artifact_process/process_file_vars/config
          else
              # Java program failed
              echo "Java preprocessing of ChEBI data failed."
              echo "FAILED=true" >> artifact_process/process_file_vars/config
          fi

      - name: Diff versions
        run: |
          chmod +x check_release_vars/config
          check_release_vars/config .
          old=$FILE_NAME
          new=mapping_preprocessing/datasources/chebi/data/$FILE_NAME
          column_name="secondaryID"
          # Extract the secondaryID column from both files and sort them
          ids_old=$(cut -f 2 "$old" | sort | tr -d "\r")
          ids_new=$(cut -f 2 "$new" | sort | tr -d "\r")

          # Perform a diff between the sorted lists of secondaryIDs
          diff_ENV=$(diff -u <(echo "$ids_old") <(echo "$ids_new"))


          # Store the output
          output_file="diff_ENV.txt"
          # Additions
          added=$(grep '^+CHEBI' "$output_file")
          echo "ADDED=$added" >> artifact_process/process_file_vars/config
          # Removals
          removed=$(grep '^-CHEBI' "$output_file")
          echo "removed=$REMOVED" >> artifact_process/process_file_vars/config
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: process_file_vars
          path: artifact_process/*
  post-issue_update:
    needs: process_file
    name: Post issue about updating data
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write
    steps:
      - name: Download check_release_vars artifact
        uses: actions/download-artifact@v4
        with:
          name: check_release_vars
      - name: Download process_file_vars artifact
        uses: actions/download-artifact@v4
        with:
          name: process_file_vars
      - uses: actions/checkout@v4
      - run: |
          ## Retrieve vars
          chmod +x check_release_vars/config
          check_release_vars/config .
          chmod +x process_file_vars/config
          process_file_vars/config .
          if [ "$DOWNLOAD_FILE" = true ]; then
            echo "---" >> issue.md
            echo "title: Update ChEBI - release on $DATE_NEW }}" >> issue.md
            echo "assignees: tabbassidaloii" >> issue.md
            echo "---" >> issue.md
            echo "[New release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE/SDF/) available from $DATE_NEW." >> issue.md
            echo ## Removed secondary IDs
            echo $REMOVED
            echo ## Added secondary IDs
            echo $ADDED
          fi
      - uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
        with:
          filename: issue.md

  post-issue-fail:
    needs: process_file
    name: Post issue about failed test
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4
      - name: Download check_release_vars artifact
        uses: actions/download-artifact@v4
        with:
          name: check_release_vars
      - name: Download process_file_vars artifact
        uses: actions/download-artifact@v4
        with:
          name: process_file_vars
      - name: Set environment variables
        run: |
          ## Retrieve vars
          chmod +x check_release_vars/config
          check_release_vars/config .
          chmod +x process_file_vars/config
          process_file_vars/config .
          if [ "$FAILED" = true ]; then
            echo "---" >> issue.md
            echo "title: Failed ChEBI processing for release $RELEASE" >> issue.md
            echo "assignees: tabbassidaloii" >> issue.md
            echo "---" >> issue.md
            echo "Processing failed for the [new release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE/SDF/) available from $DATE_NEW }}." >> issue.md
          fi
      - uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
        with:
          filename: issue.md