-
Notifications
You must be signed in to change notification settings - Fork 3
184 lines (173 loc) · 7.4 KB
/
chebi.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
name: chebi
on:
workflow_dispatch:
pull_request: # tests whether it is working on PR
paths:
- '.`/workflows/chebi.yml'
schedule:
- cron: "0 0 1,15 * *" # Run the workflow on the 1st and 15th day of each month
permissions:
contents: write
pages: write
id-token: write
jobs:
check_new_release:
runs-on: ubuntu-latest
name: Check latest release date
outputs:
RELEASE_NUMBER: ${{ steps.check_download.outputs.RELEASE_NUMBER }}
NEW_RELEASE: ${{ steps.check_download.outputs.NEW_RELEASE }}
steps:
# checkout the repository
- name: Checkout
uses: actions/checkout@v4
# check the release date for the latest ChEBI release
- name: Check for new ChEBI release
id: check_download
run: |
## Read config
. datasources/chebi/config .
echo 'Accessing the ChEBI archive'
wget https://ftp.ebi.ac.uk/pub/databases/chebi/archive/ -O chebi_index.html
## Check date for last element in index
##Extract the date from the latest release (up to the day)
date_new=$(tail -4 chebi_index.html | head -1 | grep -oP '<td align="right">\K[0-9-]+\s[0-9:]+(?=\s+</td>)' | awk '{print $1}')
release=$(tail -4 chebi_index.html | head -1 | grep -oP '(?<=a href="rel)\d\d\d')
#https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
echo "RELEASE_NUMBER=$release" >> $GITHUB_OUTPUT
##Extract the date from the ChEBI README file
date_old=$date
##Compare the dates and set variable if date_new is more recent
timestamp1=$(date -d "$date_new" +%s)
timestamp2=$(date -d "$date_old" +%s)
if [ "$timestamp1" -gt "$timestamp2" ]; then
echo 'New release available', "$release"
echo "NEW_RELEASE=true" >> $GITHUB_OUTPUT
else
echo 'No new release available'
fi
echo "Date of latest release: $date_new", "Date of release of the current version: $date_old"
##Clean up
rm chebi_index.html
test_sdf_processing:
name: Test release
if: needs.check_new_release.outputs.NEW_RELEASE
needs: check_new_release
env:
RELEASE_NUMBER: ${{ needs.check_new_release.outputs.RELEASE_NUMBER }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download SDF for new release
run: |
##Download ChEBI SDF file
echo $RELEASE_NUMBER
wget "https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel${RELEASE_NUMBER}/SDF/ChEBI_complete_3star.sdf.gz"
##Unzip gz file:
gunzip ChEBI_complete_3star.sdf.gz
##Check file size if available
ls
##Print file size
# Set up zenodo_file_id and file_name variables from config file
chmod +x datasources/chebi/config
datasources/chebi/config .
##Create temp. folder to store the data in
mkdir -p mapping_preprocessing/datasources/chebi/data
##Move the SDF file to the expected location
mv ChEBI_complete_3star.sdf mapping_preprocessing/datasources/chebi/data
# step 4: run the Java .jar for ChEBI preprocessing
- name: Set up Java
uses: actions/setup-java@v2
with:
java-version: '11'
distribution: 'adopt'
# Download current version from Zenodo
- name: Download current mapping file from Zenodo
id: download
env:
zenodo_token: ${{ secrets.ZENODO }}
run: |
# Set up zenodo_file_id and file_name variables from config file
chmod +x datasources/chebi/config
datasources/chebi/config .
file_name=$to_check_from_zenodo
echo file name: $file_name
echo 'Dowloading from: https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'
# Request Zenodo API to download the file
curl -H "Authorization: Bearer $zenodo_token" -LJO 'https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'
mv $file_name mapping_preprocessing/datasources/chebi/data/
- name: Test SDF processing
id: sdf_process
run: |
inputFile="mapping_preprocessing/datasources/chebi/data/ChEBI_complete_3star.sdf"
outputDir="mapping_preprocessing/datasources/chebi/data/"
# Run Java program and capture its exit code
java -cp target/mapping_prerocessing-0.0.1-jar-with-dependencies.jar org.sec2pri.chebi_sdf "$inputFile" "$outputDir"
# Check the exit status of the Java program
if [ $? -eq 0 ]; then
# Java program succeeded
echo "Java preprocessing of ChEBI data is done."
echo "FAILED=false" >> $GITHUB_ENV
else
# Java program failed
echo "Java preprocessing of ChEBI data failed."
echo "FAILED=true" >> $GITHUB_ENV
fi
- name: Diff versions
if:
env.FAILED == false
run: |
old=mapping_preprocessing/datasources/chebi/data/ChEBI_secID2priID.tsv"
new="mapping_preprocessing/datasources/chebi/data/new/ChEBI_secID2priID.tsv"
column_name="secondaryID"
# Extract the secondaryID column from both files and sort them
ids_old=$(cut -f 2 "$old" | sort | tr -d "\r")
ids_new=$(cut -f 2 "$new" | sort | tr -d "\r")
# Perform a diff between the sorted lists of secondaryIDs
diff_ENV=$(diff -u <(echo "$ids_old") <(echo "$ids_new"))
# Store the output
output_file="diff_ENV.txt"
# Additions
ADDED=$(grep '^+CHEBI' "$output_file")
#TODO echo "added=$ADDED" to job output
# Removals
REMOVED=$(grep '^-CHEBI' "$output_file")
echo $REMOVED > removed.txt
echo $ADDED > added.txt
- id: post_issue_new
name: Post issue for new release availability
run: |
echo "---" >> issue.md
echo "title: Update ChEBI - release on $DATE_NEW }}" >> issue.md
echo "assignees: @tabbassidaloii" >> issue.md
echo "---" >> issue.md
echo "[New release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE/SDF/) available from $DATE_NEW." >> issue.md
echo ## Removed secondary IDs
echo "```"
cat removed.txt
echo "```"
echo ## Added secondary IDs
echo "```"
cat added.txt
echo "```"
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
with:
filename: issue.md
- name: Create issue about failing test
if: env.FAILED == true
env:
RELEASE_NUMBER: ${{ needs.check_new_release.outputs.RELEASE_NUMBER }}
run: |
echo "---" >> issue.md
echo "title: Failed ChEBI processing for release $RELEASE" >> issue.md
echo "assignees: @tabbassidaloii" >> issue.md
echo "---" >> issue.md
echo "Processing failed for the [new release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE_NUMBER/SDF/) available." >> issue.md
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
with:
filename: issue.md