-
Notifications
You must be signed in to change notification settings - Fork 3
229 lines (218 loc) · 8.99 KB
/
chebi.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
name: chebi
on:
workflow_dispatch:
pull_request: # tests whether it is working on PR
paths:
- '.`/workflows/chebi.yml'
schedule:
- cron: "0 0 1,15 * *" # Run the workflow on the 1st and 15th day of each month
permissions:
contents: write
pages: write
id-token: write
jobs:
check_new_release:
runs-on: ubuntu-latest
name: Check latest release date
outputs:
release_number: ${{ steps.check_download.outputs.release_number }}
new_release: ${{ steps.check_download.outputs.new_release }}
steps:
# checkout the repository
- name: Checkout
uses: actions/checkout@v4
# check the release date for the latest ChEBI release
- name: Check for new ChEBI release
id: check_download
run: |
## Read config
. datasources/chebi/config .
echo 'Accessing the ChEBI archive'
wget https://ftp.ebi.ac.uk/pub/databases/chebi/archive/ -O chebi_index.html
## Check date for last element in index
##Extract the date from the latest release (up to the day)
date_new=$(tail -4 chebi_index.html | head -1 | grep -oP '<td align="right">\K[0-9-]+\s[0-9:]+(?=\s+</td>)' | awk '{print $1}')
release=$(tail -4 chebi_index.html | head -1 | grep -oP '(?<=a href="rel)\d\d\d')
echo "release=$release" >> "$GITHUB_OUTPUT"
##Extract the date from the ChEBI README file
date_old=$date
##Compare the dates and set variable if date_new is more recent
timestamp1=$(date -d "$date_new" +%s)
timestamp2=$(date -d "$date_old" +%s)
if [ "$timestamp1" -gt "$timestamp2" ]; then
echo 'New release available', "$release"
echo "new_release=true" > "$GITHUB_OUTPUT"
else
echo 'No new release available'
echo "new_release=false" > "$GITHUB_OUTPUT"
fi
echo "Date of latest release: $date_new", "Date of release of the current version: $date_old"
##Clean up
rm chebi_index.html
test_sdf_processing:
name: Download and test SDF processing for the latest release
needs: check_new_release
outputs:
release_number: ${{ steps.sdf_process.outputs.failed_processing }}
runs-on: ubuntu-latest
#https://stackoverflow.com/a/77876572
if: contains(needs.check_new_release.outputs.new_release, 'true')
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download SDF for new release
env:
release_number: ${{ needs.check_new_release.outputs.release_number }}
run: |
##Download ChEBI SDF file
wget https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel"$release_number"/SDF/ChEBI_complete_3star.sdf.gz
##Unzip gz file:
gunzip ChEBI_complete_3star.sdf.gz
##Check file size if available
ls
##Print file size
FILENAME=$to_check_from_zenodo
FILESIZE=$(stat -c%s "$FILENAME")
echo "Size of $FILENAME = $FILESIZE bytes."
##Create temp. folder to store the data in
mkdir -p mapping_preprocessing/datasources/chebi/data
##Move the SDF file to the expected location
mv ChEBI_complete_3star.sdf mapping_preprocessing/datasources/chebi/data
# step 4: run the Java .jar for ChEBI preprocessing
- name: Set up Java
uses: actions/setup-java@v2
with:
java-version: '11'
distribution: 'adopt'
# Download current version from Zenodo
- name: Download current mapping file from Zenodo
id: download
env:
zenodo_token: ${{ secrets.ZENODO }}
run: |
# Set up zenodo_file_id and file_name variables from config file
chmod +x datasources/chebi/config
datasources/chebi/config .
file_name=$to_check_from_zenodo
echo file name: $file_name
echo 'Dowloading from: https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'
# Request Zenodo API to download the file
curl -H "Authorization: Bearer $zenodo_token" -LJO 'https://zenodo.org/api/files/${zenodo_file_id}/${file_name}'
mv $file_name mapping_preprocessing/datasources/chebi/data/
- name: Test SDF processing
id: sdf_process
run: |
inputFile="mapping_preprocessing/datasources/chebi/data/ChEBI_complete_3star.sdf"
outputDir="mapping_preprocessing/datasources/chebi/data/"
# Run Java program and capture its exit code
java -cp target/mapping_prerocessing-0.0.1-jar-with-dependencies.jar org.sec2pri.chebi_sdf "$inputFile" "$outputDir"
# Check the exit status of the Java program
if [ $? -eq 0 ]; then
# Java program succeeded
echo "Java preprocessing of ChEBI data is done."
echo "failed_processing=false" >> "$GITHUB_OUTPUT"
#TODO echo "FAILED=false" to job output
else
# Java program failed
echo "Java preprocessing of ChEBI data failed."
echo "failed_processing=true" >> "$GITHUB_OUTPUT"
fi
- name: Diff versions
run: |
TODO old=mapping_preprocessing/datasources/chebi/data/ChEBI_secID2priID.tsv"
new="mapping_preprocessing/datasources/chebi/data/new/ChEBI_secID2priID.tsv"
column_name="secondaryID"
# Extract the secondaryID column from both files and sort them
ids_old=$(cut -f 2 "$old" | sort | tr -d "\r")
ids_new=$(cut -f 2 "$new" | sort | tr -d "\r")
# Perform a diff between the sorted lists of secondaryIDs
diff_ENV=$(diff -u <(echo "$ids_old") <(echo "$ids_new"))
# Store the output
output_file="diff_ENV.txt"
# Additions
ADDED=$(grep '^+CHEBI' "$output_file")
#TODO echo "added=$ADDED" to job output
# Removals
REMOVED=$(grep '^-CHEBI' "$output_file")
#TODO echo "removed=$REMOVED" to job output
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: test_sdf_processing_vars
path: artifact_process/*
post-issue_update:
needs: [test_sdf_processing, check_new_release]
name: Post issue about updating data
if: contains(needs.check_new_release.outputs.new_release, 'true')
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- name: Download check_release_vars artifact
uses: actions/download-artifact@v4
with:
name: check_release_vars
- name: Download test_sdf_processing_vars artifact
uses: actions/download-artifact@v4
with:
name: test_sdf_processing_vars
- uses: actions/checkout@v4
- run: |
## Retrieve vars
chmod +x check_release_vars/config
check_release_vars/config .
#TODO retrieve output from last job
# RELEASE=
chmod +x test_sdf_processing_vars/config
test_sdf_processing_vars/config .
if [ "$DOWNLOAD_FILE" = true ]; then
echo "---" >> issue.md
echo "title: Update ChEBI - release on $DATE_NEW }}" >> issue.md
echo "assignees: @tabbassidaloii" >> issue.md
echo "---" >> issue.md
echo "[New release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$RELEASE/SDF/) available from $DATE_NEW." >> issue.md
echo ## Removed secondary IDs
echo $REMOVED
echo ## Added secondary IDs
echo $ADDED
fi
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
with:
filename: issue.md
post-issue-fail:
needs: test_sdf_processing
if: contains(needs.test_sdf_processing.outputs.failed_processing, 'true')
name: Post issue about failed test
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@v4
- name: Download check_release_vars artifact
uses: actions/download-artifact@v4
with:
name: check_release_vars
- name: Download test_sdf_processing_vars artifact
uses: actions/download-artifact@v4
with:
name: test_sdf_processing_vars
- name: Create issue
env:
release_number: ${{ needs.check_new_release.outputs.release_number }}
run: |
if [ "$FAILED" = true ]; then
echo "---" >> issue.md
echo "title: Failed ChEBI processing for release $RELEASE" >> issue.md
echo "assignees: @tabbassidaloii" >> issue.md
echo "---" >> issue.md
echo "Processing failed for the [new release for ChEBI](https://ftp.ebi.ac.uk/pub/databases/chebi/archive/rel$release_number/SDF/) available." >> issue.md
fi
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: secrets.GITHUB_TOKEN }}
with:
filename: issue.md