diff --git a/certs/ydb_crypt_config.libconfig b/certs/ydb_crypt_config.libconfig new file mode 100644 index 0000000..dbda9d0 --- /dev/null +++ b/certs/ydb_crypt_config.libconfig @@ -0,0 +1,9 @@ +tls: { + dev: { + format: "PEM"; + cert: "/data/certs/mycert.pem"; + key: "/data/certs/mycert.key"; + ssl-options: "SSL_OP_NO_SSLv2:SSL_OP_NO_SSLv3:SSL_OP_NO_TLSv1:SSL_OP_NO_TLSv1_1"; + cipher-list: "ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:!aNULL:!SHA1:!AESCCM"; + }; +} \ No newline at end of file diff --git a/compose/compose.yaml b/compose/compose.yaml new file mode 100644 index 0000000..0c81359 --- /dev/null +++ b/compose/compose.yaml @@ -0,0 +1,12 @@ +services: + db: + image: harbor.ukserp.ac.uk/linkage/assign:pr-8 + ports: + - "9080:9080" + - "9081:9081" + volumes: + - type: volume + source: db-data + target: /data +volumes: + db-data: \ No newline at end of file diff --git a/containers/assign/Dockerfile b/containers/assign/Dockerfile index 47d7177..584f127 100644 --- a/containers/assign/Dockerfile +++ b/containers/assign/Dockerfile @@ -1,27 +1,59 @@ -FROM yottadb/yottadb-base +ARG YDB_VERSION="r2.01" + +FROM yottadb/yottadb:${YDB_VERSION} LABEL org.opencontainers.image.source=https://github.com/SwanseaUniversityMedical/ASSIGN-container -USER root +#USER root SHELL ["/bin/bash", "-c"] # Get git. RUN apt-get update && apt-get install -y git +# ASSIGN install env vars ENV assign_url="https://github.com/endeavourhealth-discovery/ASSIGN.git" ENV assign_dest="/data/ASSIGN" ENV assign_sha="" ENV abp_dir="/data/ABP" -ENV ydb_chset=M + +# YDB env vars +ARG YDB_VERSION +ENV ydb_chset="M" +ENV ydb_dir="/data" +ENV ydb_rel="${YDB_VERSION}_x86_64" # Create directory to hold ABP RUN mkdir -p $abp_dir +# Yotta TLS setup +RUN apt-get update && \ + apt-get install -y make curl wget gcc libgcrypt-dev libssl-dev libgpgme-dev libconfig-dev tcsh && \ + /opt/yottadb/current/ydbinstall --plugins-only --encplugin + +# Placeholder for testing only +ENV USER=root +#ENV cert_pass="monkey1234" +#ENV cert_subj="/C=US/ST=Washington/L=Seattle/CN=www.smh101.com" +#ENV cert_dir="/data/certs" + +#RUN mkdir -p $cert_dir &&\ +# openssl genrsa -aes128 -passout pass:$cert_pass -out $cert_dir/mycert.key 2048 && \ +# openssl req -new -key $cert_dir/mycert.key -passin pass:$cert_pass -subj $cert_subj -out $cert_dir/mycert.csr && \ +# openssl req -x509 -days 365 -sha256 -in $cert_dir/mycert.csr -key $cert_dir/mycert.key -passin pass:$cert_pass -out $cert_dir/mycert.pem + +#COPY certs/ydb_crypt_config.libconfig $cert_dir/ydb_crypt_config.libconfig +#ENV ydb_crypt_config="$cert_dir/ydb_crypt_config.libconfig" + # Add in startup script. COPY startup_scripts/assign-startup.sh /assign-startup.sh RUN chmod +x /assign-startup.sh +# Add in web auth +COPY startup_scripts/ADDWEBAUTH.m /extra_scripts/ADDWEBAUTH.m +RUN chmod 644 /extra_scripts/ADDWEBAUTH.m + # Expose port for YottaDB GUI -EXPOSE 9080/tcp +EXPOSE 9080 +EXPOSE 9081 ENTRYPOINT ["/assign-startup.sh"] \ No newline at end of file diff --git a/startup_scripts/ADDWEBAUTH.m b/startup_scripts/ADDWEBAUTH.m new file mode 100644 index 0000000..44c8fb1 --- /dev/null +++ b/startup_scripts/ADDWEBAUTH.m @@ -0,0 +1,9 @@ +ADDWEBAUTH ; ; 15/11/24 + s keyvalue="FAKE_KEY" ; bring these in from vault? + s username="user" + s userpass="password" + + s ^ICONFIG("KEY")=keyvalue + s Y=$$TORCFOUR^EWEBRC4(userpass,^ICONFIG("KEY")) + s ^BUSER("USER",username)=Y + q \ No newline at end of file diff --git a/startup_scripts/assign-startup.sh b/startup_scripts/assign-startup.sh index 44dd133..081c56c 100644 --- a/startup_scripts/assign-startup.sh +++ b/startup_scripts/assign-startup.sh @@ -2,36 +2,86 @@ mkdir -p /data/logs -# Set env vars -. /opt/yottadb/current/ydb_env_set +# Check if database file exists, if so try a rundown incase it was borked by container stopping +if [ -f "$ydb_dir/$ydb_rel/g/yottadb.gld" ]; then + echo "Running rundown to restore database." + export ydb_gbldir="$ydb_dir/$ydb_rel/g/yottadb.gld" + /opt/yottadb/current/mupip rundown -region DEFAULT +fi + +# Check if env vars need setting (not ideal, but better than checking every one +if [ -z "${ydb_dist}" ]; then + echo "ydb_dist not set. Setting variables." + . /opt/yottadb/current/ydb_env_set +else + echo "ydb_dist already set. Not setting variables again." +fi -# Get ASSIGN +# Check if ASSIGN needs pulling if [ ! -d "$assign_dest" ]; then echo "Obtaining ASSIGN routines." + # Clone the wanted sha from github - git clone $assign_url $assign_dest - git -C $assign_dest checkout $assign_sha + if [ -z "${assign_sha}" ]; then + git clone $assign_url $assign_dest --depth 1 + else + git clone $assign_url $assign_dest --depth 1 --branch $assign_sha + fi # Put the routines to the YottaDB routines directory echo "Moving the routines." cp $assign_dest/UPRN/yottadb/* $ydb_dir/$ydb_rel/r cp $assign_dest/UPRN/codelists/* $abp_dir - - # Perform zlink of routines - echo "Linking ASSIGN routines." - $ydb_dist/ydb -run ^ZLINK else echo "$assign_dest already exists. Not cloning down ASSIGN again." fi +# Perform zlink of routines, doesn't matter if already linked +echo "Linking ASSIGN routines, you may see warnings." +$ydb_dist/ydb -run ^ZLINK +echo "Routines linked." + # Update YottaDB database settings $ydb_dist/mupip set -NULL_SUBSCRIPTS=true -region DEFAULT && \ -$ydb_dist/mupip set -journal=off -region DEFAULT && \ $ydb_dist/mupip set -extension_count=500000 -region DEFAULT && \ -$ydb_dist/mupip set -access_method=mm -region DEFAULT +$ydb_dist/mupip set -journal=off -region DEFAULT #&& \ +#$ydb_dist/mupip set -access_method=mm -region DEFAULT -# Startup webgui -yottadb -run %ydbgui --read --port 9080 >>/data/logs/%ydbgui.log & +# Do data ingest, look at checksum for "$ydb_dir/$ydb_rel/g/yottadb.gld" +export checksum_loc=/data/import_checksum +if [ ! -f "/data/import_checksum" ]; then + echo "Ingesting ABP from $abp_dir" + $ydb_dist/ydb -run %XCMD 'd IMPORT^UPRN1A("/data/ABP")' + echo "Producing checksum for $ydb_gbldir" + sha256sum $ydb_gbldir | awk '{ print $1 }' > $checksum_loc + cat /data/import_checksum +else + echo "Get checksums" + prev_checksum=$(cat $checksum_loc) + cur_checksum=$(shasum -a 256 $ydb_gbldir | awk '{ print $1 }') + echo "Previous checksum: $prev_checksum" + echo "Current checksum: $cur_checksum" + if [[ $prev_checksum != $cur_checksum ]]; then + echo "Reproducing checksum for $ydb_gbldir" + sha256sum $ydb_gbldir | awk '{ print $1 }' > $checksum_loc + fi +fi + +# Set the ybd env var for the TLS password hash, this can be replaced with an ENV var in the image? +#export ydb_tls_passwd_dev="$($ydb_dist/plugin/ydbcrypt/maskpass <<< $cert_pass | cut -d ":" -f2 | tr -d ' ')" -# Drop in to ydb -exec $ydb_dist/yottadb -direct +# Startup the ASSIGN TLS listener +echo "Starting listener" +yottadb -run %XCMD 'job START^VPRJREQ(9081)' & + +# Startup the ASSIGN web API interface +echo "Starting ASSIGN API endpoints" +cp "/extra_scripts/ADDWEBAUTH.m" "$ydb_dir/$ydb_rel/r/ADDWEBAUTH.m" +yottadb -run INT^VUE # File upload/download +yottadb -run SETUP^UPRNHOOK2 # UPRN retrieval +yottadb -run ^NEL # request handling +yottadb -run ^ADDWEBAUTH # Add creds + +# Startup webgui +echo "Starting YottaDB GUI endpoint" +yottadb -run %ydbgui --readwrite --port 9080 >>/data/logs/%ydbgui.log diff --git a/testing_scripts/check_batch_uprn.py b/testing_scripts/check_batch_uprn.py new file mode 100644 index 0000000..f5a5e2b --- /dev/null +++ b/testing_scripts/check_batch_uprn.py @@ -0,0 +1,27 @@ +import requests +import os + +endpoint = 'http://localhost:9081' +username = 'user' +password = 'password' +filepath = './' +filename = 'test_in.tsv' +processed_filename = 'test_out.txt' + +upload_url = f'{endpoint}/api2/fileupload2' +download_url = f'{endpoint}/api2/filedownload2?filename={filename}' + +# Test upload +with open(os.path.join(filepath, filename), 'rb') as f: + files = {'file': (filename, f, 'text/plain')} + r = requests.post(upload_url, files=files, auth=(username, password)) +print(r.text) +print(r.status_code) +assert r.status_code == 201 + +# Test download of processed file +r = requests.get(download_url, auth=(username, password)) +with open(os.path.join(filepath, processed_filename), "w") as f: + f.write(r.text) +print(r.status_code) +assert r.status_code == 200 diff --git a/testing_scripts/check_single_query.py b/testing_scripts/check_single_query.py new file mode 100644 index 0000000..ce5b901 --- /dev/null +++ b/testing_scripts/check_single_query.py @@ -0,0 +1,25 @@ +import requests +import json + +endpoint = 'http://localhost:9081' +username = 'user' +password = 'password' + +# Testing address +address_string = 'Swansea University,Swansea,SA2 8PP'.replace(' ', '+') +address_uprn = '10010023401' + +address_query_url = f'{endpoint}/api2/getinfo?adrec={address_string}' +uprn_query_url = f'{endpoint}/api2/getuprn?uprn={address_uprn}' + +# Test UPRN lookup via address string +r = requests.get(address_query_url, auth=(username, password)) +print(r.text) +assert r.status_code == 200 +assert(json.loads(r.text)['BestMatch']['UPRN'] == address_uprn) + +# Test address lookup via UPRN string +r = requests.get(uprn_query_url, auth=(username, password)) +print(r.text) +assert r.status_code == 200 +assert(json.loads(r.text)['UPRN'] == address_uprn) \ No newline at end of file diff --git a/testing_scripts/test_in.tsv b/testing_scripts/test_in.tsv new file mode 100644 index 0000000..c09878e --- /dev/null +++ b/testing_scripts/test_in.tsv @@ -0,0 +1 @@ +1 Swansea University,Swansea,SA2 8PP \ No newline at end of file