feat: Added TLS setup and api service

certs/ydb_crypt_config.libconfig

@@ -0,0 +1,9 @@
+tls: {
+  dev: {
+    format: "PEM";
+    cert: "/data/certs/mycert.pem";
+    key:  "/data/certs/mycert.key";
+    ssl-options: "SSL_OP_NO_SSLv2:SSL_OP_NO_SSLv3:SSL_OP_NO_TLSv1:SSL_OP_NO_TLSv1_1";
+    cipher-list: "ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:!aNULL:!SHA1:!AESCCM";
+  };
+}

compose/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  db:
+    image: harbor.ukserp.ac.uk/linkage/assign:pr-8
+    ports:
+      - "9080:9080"
+      - "9081:9081"
+    volumes:
+      - type: volume
+        source: db-data
+        target: /data
+volumes:
+  db-data:

containers/assign/Dockerfile

@@ -1,27 +1,59 @@
-FROM yottadb/yottadb-base
+ARG YDB_VERSION="r2.01"
+
+FROM yottadb/yottadb:${YDB_VERSION}
 
 LABEL org.opencontainers.image.source=https://github.com/SwanseaUniversityMedical/ASSIGN-container
 
-USER root
+#USER root
 SHELL ["/bin/bash", "-c"]
 
 # Get git.
 RUN apt-get update && apt-get install -y git
 
+# ASSIGN install env vars
 ENV assign_url="https://github.com/endeavourhealth-discovery/ASSIGN.git"
 ENV assign_dest="/data/ASSIGN"
 ENV assign_sha=""
 ENV abp_dir="/data/ABP"
-ENV ydb_chset=M
+
+# YDB env vars
+ARG YDB_VERSION
+ENV ydb_chset="M"
+ENV ydb_dir="/data"
+ENV ydb_rel="${YDB_VERSION}_x86_64"
 
 # Create directory to hold ABP
 RUN mkdir -p $abp_dir
 
+# Yotta TLS setup
+RUN apt-get update && \
+    apt-get install -y make curl wget gcc libgcrypt-dev libssl-dev libgpgme-dev libconfig-dev tcsh && \
+    /opt/yottadb/current/ydbinstall --plugins-only --encplugin
+
+# Placeholder for testing only
+ENV USER=root
+#ENV cert_pass="monkey1234"
+#ENV cert_subj="/C=US/ST=Washington/L=Seattle/CN=www.smh101.com"
+#ENV cert_dir="/data/certs"
+
+#RUN mkdir -p $cert_dir &&\
+#    openssl genrsa -aes128 -passout pass:$cert_pass -out $cert_dir/mycert.key 2048 && \
+#    openssl req -new -key $cert_dir/mycert.key -passin pass:$cert_pass -subj $cert_subj -out $cert_dir/mycert.csr && \
+#    openssl req -x509 -days 365 -sha256 -in $cert_dir/mycert.csr -key $cert_dir/mycert.key -passin pass:$cert_pass -out $cert_dir/mycert.pem
+
+#COPY certs/ydb_crypt_config.libconfig $cert_dir/ydb_crypt_config.libconfig
+#ENV ydb_crypt_config="$cert_dir/ydb_crypt_config.libconfig"
+
 # Add in startup script.
 COPY startup_scripts/assign-startup.sh /assign-startup.sh
 RUN chmod +x /assign-startup.sh
 
+# Add in web auth
+COPY startup_scripts/ADDWEBAUTH.m /extra_scripts/ADDWEBAUTH.m
+RUN chmod 644 /extra_scripts/ADDWEBAUTH.m
+
 # Expose port for YottaDB GUI
-EXPOSE 9080/tcp
+EXPOSE 9080
+EXPOSE 9081
 
 ENTRYPOINT ["/assign-startup.sh"]

startup_scripts/ADDWEBAUTH.m

@@ -0,0 +1,9 @@
+ADDWEBAUTH ; ; 15/11/24
+ s keyvalue="FAKE_KEY" ; bring these in from vault?
+ s username="user"
+ s userpass="password"
+
+ s ^ICONFIG("KEY")=keyvalue
+ s Y=$$TORCFOUR^EWEBRC4(userpass,^ICONFIG("KEY"))
+ s ^BUSER("USER",username)=Y
+ q

startup_scripts/assign-startup.sh

@@ -2,36 +2,86 @@
 
 mkdir -p /data/logs
 
-# Set env vars
-. /opt/yottadb/current/ydb_env_set
+# Check if database file exists, if so try a rundown incase it was borked by container stopping
+if [ -f "$ydb_dir/$ydb_rel/g/yottadb.gld" ]; then
+  echo "Running rundown to restore database."
+  export ydb_gbldir="$ydb_dir/$ydb_rel/g/yottadb.gld"
+  /opt/yottadb/current/mupip rundown -region DEFAULT
+fi
+
+# Check if env vars need setting (not ideal, but better than checking every one
+if [ -z "${ydb_dist}" ]; then
+  echo "ydb_dist not set. Setting variables."
+  . /opt/yottadb/current/ydb_env_set
+else
+  echo "ydb_dist already set. Not setting variables again."
+fi
 
-# Get ASSIGN
+# Check if ASSIGN needs pulling
 if [ ! -d "$assign_dest" ]; then
   echo "Obtaining ASSIGN routines."
+
   # Clone the wanted sha from github
-  git clone $assign_url $assign_dest
-  git -C $assign_dest checkout $assign_sha
+    if [ -z "${assign_sha}" ]; then
+      git clone $assign_url $assign_dest --depth 1
+    else
+      git clone $assign_url $assign_dest --depth 1 --branch $assign_sha
+    fi
 
   # Put the routines to the YottaDB routines directory
   echo "Moving the routines."
   cp $assign_dest/UPRN/yottadb/* $ydb_dir/$ydb_rel/r
   cp $assign_dest/UPRN/codelists/* $abp_dir
-
-  # Perform zlink of routines
-  echo "Linking ASSIGN routines."
-  $ydb_dist/ydb -run ^ZLINK
 else
   echo "$assign_dest already exists. Not cloning down ASSIGN again."
 fi
 
+# Perform zlink of routines, doesn't matter if already linked
+echo "Linking ASSIGN routines, you may see warnings."
+$ydb_dist/ydb -run ^ZLINK
+echo "Routines linked."
+
 # Update YottaDB database settings
 $ydb_dist/mupip set -NULL_SUBSCRIPTS=true -region DEFAULT && \
-$ydb_dist/mupip set -journal=off -region DEFAULT && \
 $ydb_dist/mupip set -extension_count=500000 -region DEFAULT && \
-$ydb_dist/mupip set -access_method=mm -region DEFAULT
+$ydb_dist/mupip set -journal=off -region DEFAULT #&& \
+#$ydb_dist/mupip set -access_method=mm -region DEFAULT
 
-# Startup webgui
-yottadb -run %ydbgui --read --port 9080 >>/data/logs/%ydbgui.log &
+# Do data ingest, look at checksum for "$ydb_dir/$ydb_rel/g/yottadb.gld"
+export checksum_loc=/data/import_checksum
+if [ ! -f "/data/import_checksum" ]; then
+  echo "Ingesting ABP from $abp_dir"
+  $ydb_dist/ydb -run %XCMD 'd IMPORT^UPRN1A("/data/ABP")'
+  echo "Producing checksum for $ydb_gbldir"
+  sha256sum $ydb_gbldir | awk '{ print $1 }' > $checksum_loc
+  cat /data/import_checksum
+else
+  echo "Get checksums"
+  prev_checksum=$(cat $checksum_loc)
+  cur_checksum=$(shasum -a 256 $ydb_gbldir | awk '{ print $1 }')
+  echo "Previous checksum: $prev_checksum"
+  echo "Current checksum: $cur_checksum"
+  if [[ $prev_checksum != $cur_checksum ]]; then
+    echo "Reproducing checksum for $ydb_gbldir"
+    sha256sum $ydb_gbldir | awk '{ print $1 }' > $checksum_loc
+  fi
+fi
+
+# Set the ybd env var for the TLS password hash, this can be replaced with an ENV var in the image?
+#export ydb_tls_passwd_dev="$($ydb_dist/plugin/ydbcrypt/maskpass <<< $cert_pass | cut -d ":" -f2 | tr -d ' ')"
 
-# Drop in to ydb
-exec $ydb_dist/yottadb -direct
+# Startup the ASSIGN TLS listener
+echo "Starting listener"
+yottadb -run %XCMD 'job START^VPRJREQ(9081)' &
+
+# Startup the ASSIGN web API interface
+echo "Starting ASSIGN API endpoints"
+cp "/extra_scripts/ADDWEBAUTH.m" "$ydb_dir/$ydb_rel/r/ADDWEBAUTH.m"
+yottadb -run INT^VUE          # File upload/download
+yottadb -run SETUP^UPRNHOOK2  # UPRN retrieval
+yottadb -run ^NEL             # request handling
+yottadb -run ^ADDWEBAUTH      # Add creds
+
+# Startup webgui
+echo "Starting YottaDB GUI endpoint"
+yottadb -run %ydbgui --readwrite --port 9080 >>/data/logs/%ydbgui.log

testing_scripts/check_batch_uprn.py

@@ -0,0 +1,27 @@
+import requests
+import os
+
+endpoint = 'http://localhost:9081'
+username = 'user'
+password = 'password'
+filepath = './'
+filename = 'test_in.tsv'
+processed_filename = 'test_out.txt'
+
+upload_url = f'{endpoint}/api2/fileupload2'
+download_url = f'{endpoint}/api2/filedownload2?filename={filename}'
+
+# Test upload
+with open(os.path.join(filepath, filename), 'rb') as f:
+    files = {'file': (filename, f, 'text/plain')}
+    r = requests.post(upload_url, files=files, auth=(username, password))
+print(r.text)
+print(r.status_code)
+assert r.status_code == 201
+
+# Test download of processed file
+r = requests.get(download_url, auth=(username, password))
+with open(os.path.join(filepath, processed_filename), "w") as f:
+    f.write(r.text)
+print(r.status_code)
+assert r.status_code == 200

testing_scripts/check_single_query.py

@@ -0,0 +1,25 @@
+import requests
+import json
+
+endpoint = 'http://localhost:9081'
+username = 'user'
+password = 'password'
+
+# Testing address
+address_string = 'Swansea University,Swansea,SA2 8PP'.replace(' ', '+')
+address_uprn = '10010023401'
+
+address_query_url = f'{endpoint}/api2/getinfo?adrec={address_string}'
+uprn_query_url = f'{endpoint}/api2/getuprn?uprn={address_uprn}'
+
+# Test UPRN lookup via address string
+r = requests.get(address_query_url, auth=(username, password))
+print(r.text)
+assert r.status_code == 200
+assert(json.loads(r.text)['BestMatch']['UPRN'] == address_uprn)
+
+# Test address lookup via UPRN string
+r = requests.get(uprn_query_url, auth=(username, password))
+print(r.text)
+assert r.status_code == 200
+assert(json.loads(r.text)['UPRN'] == address_uprn)

testing_scripts/test_in.tsv

@@ -0,0 +1 @@
+1	Swansea University,Swansea,SA2 8PP