From 48a226aaadf0d003a60926b24a397038a5194ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gru=CC=88ndner?= Date: Fri, 1 Jul 2022 14:52:40 +0200 Subject: [PATCH 1/5] init update to mii deploy and refactor --- deploy/create_install_packages.sh | 8 --- deploy/opal/.env.default | 12 ++++ deploy/{opal_poll => opal}/README.md | 0 deploy/opal/auth/import_certificate.sh | 3 + deploy/opal/docker-compose.yml | 38 ++++++++++++ deploy/opal/start.sh | 26 ++++++++ deploy/{opal_poll => opal}/stop.sh | 2 +- deploy/opal_poll/.env | 27 --------- deploy/opal_poll/auth/import_certificate.sh | 7 --- deploy/opal_poll/docker-compose.yml | 59 ------------------- deploy/opal_poll/install.sh | 39 ------------ deploy/opal_poll/scripts/addCaCertificates.sh | 4 -- .../scripts/add_miracum_import_user.sh | 11 ---- .../opal_poll/scripts/add_miracum_projects.sh | 14 ----- deploy/opal_poll/scripts/add_miracum_users.sh | 19 ------ deploy/opal_poll/scripts/add_user_to_table.sh | 10 ---- .../opal_poll/scripts/import_certificate.sh | 4 -- deploy/opal_poll/scripts/poll_admin.sh | 28 --------- deploy/opal_poll/update.sh | 48 --------------- deploy/poll/.env.default | 13 ++++ deploy/poll/docker-compose.yml | 13 ++++ deploy/{opal_poll => poll}/start.sh | 4 +- deploy/poll/stop.sh | 5 ++ deploy/queue/{.env => .env.default} | 2 +- deploy/queue/{nginx => auth}/nginx.conf | 3 +- deploy/queue/docker-compose.yml | 24 +++----- deploy/queue/install.sh | 54 ----------------- deploy/queue/nginx/mime.types | 48 --------------- deploy/queue/nginx/q_nginx_start.sh | 20 ------- deploy/queue/start.sh | 30 ++++++++++ deploy/queue/update.sh | 39 ------------ deploy/test.py | 5 -- deploy/test/.env | 10 ---- deploy/test/.env.default | 2 + deploy/test/docker-compose.yml | 8 +++ deploy/test/ds_test.sh | 0 ds_queue/nginx/nginx.conf | 5 ++ 37 files changed, 170 insertions(+), 474 deletions(-) delete mode 100755 deploy/create_install_packages.sh create mode 100755 deploy/opal/.env.default rename deploy/{opal_poll => opal}/README.md (100%) create mode 100755 deploy/opal/auth/import_certificate.sh create mode 100644 deploy/opal/docker-compose.yml create mode 100755 deploy/opal/start.sh rename deploy/{opal_poll => opal}/stop.sh (73%) mode change 100755 => 100644 delete mode 100755 deploy/opal_poll/.env delete mode 100755 deploy/opal_poll/auth/import_certificate.sh delete mode 100644 deploy/opal_poll/docker-compose.yml delete mode 100755 deploy/opal_poll/install.sh delete mode 100755 deploy/opal_poll/scripts/addCaCertificates.sh delete mode 100644 deploy/opal_poll/scripts/add_miracum_import_user.sh delete mode 100644 deploy/opal_poll/scripts/add_miracum_projects.sh delete mode 100644 deploy/opal_poll/scripts/add_miracum_users.sh delete mode 100644 deploy/opal_poll/scripts/add_user_to_table.sh delete mode 100644 deploy/opal_poll/scripts/import_certificate.sh delete mode 100755 deploy/opal_poll/scripts/poll_admin.sh delete mode 100755 deploy/opal_poll/update.sh create mode 100755 deploy/poll/.env.default create mode 100644 deploy/poll/docker-compose.yml rename deploy/{opal_poll => poll}/start.sh (93%) create mode 100644 deploy/poll/stop.sh rename deploy/queue/{.env => .env.default} (97%) rename deploy/queue/{nginx => auth}/nginx.conf (99%) delete mode 100755 deploy/queue/install.sh delete mode 100644 deploy/queue/nginx/mime.types delete mode 100755 deploy/queue/nginx/q_nginx_start.sh create mode 100755 deploy/queue/start.sh delete mode 100755 deploy/queue/update.sh delete mode 100644 deploy/test.py delete mode 100755 deploy/test/.env create mode 100755 deploy/test/.env.default create mode 100644 deploy/test/docker-compose.yml mode change 100755 => 100644 deploy/test/ds_test.sh diff --git a/deploy/create_install_packages.sh b/deploy/create_install_packages.sh deleted file mode 100755 index bd93e91..0000000 --- a/deploy/create_install_packages.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -TAG=${1:-"latest"} - -cd opal_poll && tar cvzf "../opal_poll_install_$TAG.tgz" * .env -cd ../queue && tar cvzf "../queue_install_$TAG.tgz" * .env -cd ../analysis && tar cvzf "../analysis_install_$TAG.tgz" * .env -cd ../test && tar cvzf "../test_install_$TAG.tgz" * .env \ No newline at end of file diff --git a/deploy/opal/.env.default b/deploy/opal/.env.default new file mode 100755 index 0000000..6a1c8b7 --- /dev/null +++ b/deploy/opal/.env.default @@ -0,0 +1,12 @@ +# this file lets you install opal and the poll service + +### add your configuration information here +QP_DOCKER_PROJECT=ds-qp +QP_HOME_DIR= +QP_DATA_DIR=/opt/dsqp +# please change this password to a save password +OPAL_ADMIN_PASS=password +OPAL_ROCKS_HOSTS= +OPAL_MONGODB_HOST= +DS_VERSION=6.0.0 +DS_PRIVACY_LEVEL=5 \ No newline at end of file diff --git a/deploy/opal_poll/README.md b/deploy/opal/README.md similarity index 100% rename from deploy/opal_poll/README.md rename to deploy/opal/README.md diff --git a/deploy/opal/auth/import_certificate.sh b/deploy/opal/auth/import_certificate.sh new file mode 100755 index 0000000..8bf305f --- /dev/null +++ b/deploy/opal/auth/import_certificate.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec datashield_opal bash -c "cd /opt/opal/bin && ./import_opal_cert.sh" \ No newline at end of file diff --git a/deploy/opal/docker-compose.yml b/deploy/opal/docker-compose.yml new file mode 100644 index 0000000..1e4dc09 --- /dev/null +++ b/deploy/opal/docker-compose.yml @@ -0,0 +1,38 @@ +services: + opal: + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_opal:sha256-5de21239f4c8987728c840c2de8ba8d76d386f59d1ec9d9a3626ac21b25b40b6.sig + restart: unless-stopped + ports: + - "8443:8443" + - 127.0.0.1:8880:8080 + environment: + - OPAL_ADMINISTRATOR_PASSWORD=${OPAL_ADMIN_PASS} + - MONGODBHOST=${OPAL_MONGODB_HOST:-mongo} + - INITTESTDATA=${OPAL_INITTESTDATA:-true} + - DS_VERSION=${DS_VERSION:-6.1.1} + - DS_PRIVACY_LEVEL=${DS_PRIVACY_LEVEL:-5} + - ROCK_HOSTS=${OPAL_ROCKS_HOSTS:-rock1:8085,rock2:8085} + - JAVA_OPTS=-Xms1G -Xmx8G -XX:+UseG1GC + volumes: + - ./auth:/opt/opal/auth + mongo: + image: mongo + restart: unless-stopped + volumes: + - opal-db-data:/data/db + rock1: + image: datashield/rock-base:latest + restart: unless-stopped + environment: + - ROCK_ID=default-1 + - ROCK_CLUSTER=default + rock2: + image: datashield/rock-base:latest + restart: unless-stopped + environment: + - ROCK_ID=default-2 + - ROCK_CLUSTER=default + +volumes: + opal-db-data: + opal-files: diff --git a/deploy/opal/start.sh b/deploy/opal/start.sh new file mode 100755 index 0000000..7afc6a1 --- /dev/null +++ b/deploy/opal/start.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +source .env + +if [[ $(which docker) && $(which docker-compose) ]]; then + echo "docker and docker compose already installed, versions are: " + docker -v + docker-compose -v +else + echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" + echo "aborting installation" + exit +fi + +KEY_FILE=auth/key.pem +CERT_FILE=auth/cert.pem +if [ ! -f "$KEY_FILE" ] || [ ! -f "$CERT_FILE" ]; then + echo "ERROR missing certificates for nginx proxy" + echo "please create a $CERT_FILE and $KEY_FILE accordingly" + echo "aborting installation" + exit +fi + +docker-compose up -p $QP_DOCKER_PROJECT -d + +printf "\n - Check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file diff --git a/deploy/opal_poll/stop.sh b/deploy/opal/stop.sh old mode 100755 new mode 100644 similarity index 73% rename from deploy/opal_poll/stop.sh rename to deploy/opal/stop.sh index 29e0269..afc4079 --- a/deploy/opal_poll/stop.sh +++ b/deploy/opal/stop.sh @@ -1,4 +1,4 @@ #!/bin/bash printf "######################\Stopping Opal, Datashield and RServer ...\n######################\n\n" -docker-compose stop +docker-compose -p $QP_DOCKER_PROJECT stop \ No newline at end of file diff --git a/deploy/opal_poll/.env b/deploy/opal_poll/.env deleted file mode 100755 index 984dddb..0000000 --- a/deploy/opal_poll/.env +++ /dev/null @@ -1,27 +0,0 @@ -# this file lets you install opal and the poll service - -### add your configuration information here - -# Ip address of this server - default is localhost -OPAL_SERVER_IP=127.0.0.1 - -QP_HOME_DIR= -QP_DATA_DIR=/opt/dsqp -# please change this password to a save password -OPAL_ADMIN_PASS=password -R_SERVER_HOST=datashield_rserver -OPAL_MONGODB_HOST=datashield_mongo -DS_VERSION=6.0.0 -DS_PRIVACY_LEVEL=5 - -# e.g. 123.12.12.12:443 , queue server host and port(usually 443) -POLL_QUEUE_SERVER= -# e.g. '123.12.12.12:443' , opal server host and port, default is set to datashield_opal:8443, which is correct for a pure docker setup -POLL_OPAL_SERVER=datashield_opal:8443 -# e.g. 5 number of poll threads default 2 -POLL_THREADS=2 -LOG_LEVEL=20 - # e.g. '-c' activate checking sever certificate -CHECK_SERVER_CERT= -MIRACUM_PROJECTS=("miracum_tca" "miracum_hb") -MIRACUM_USERS=("miracum_er_1" "miracum_er_2" "miracum_er_3" "miracum_fr_1" "miracum_fr_2" "miracum_fr_3") \ No newline at end of file diff --git a/deploy/opal_poll/auth/import_certificate.sh b/deploy/opal_poll/auth/import_certificate.sh deleted file mode 100755 index 3c43327..0000000 --- a/deploy/opal_poll/auth/import_certificate.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -OPAL_ADMIN_PASS=${OPAL_ADMINISTRATOR_PASSWORD:-"password"} - -OPAL_KEY=$(sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g' opalkey.pem) -OPAL_CERT=$(sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g' opalcert.pem) - -echo "{\"alias\":\"https\",\"keyType\":\"KEY_PAIR\", \"privateImport\": \"$OPAL_KEY\", \"publicImport\" :\"$OPAL_CERT\"}" | opal rest --opal https://localhost:8443 -u administrator -p $OPAL_ADMIN_PASS --content-type 'application/json' -m PUT /system/keystore diff --git a/deploy/opal_poll/docker-compose.yml b/deploy/opal_poll/docker-compose.yml deleted file mode 100644 index 22c6be4..0000000 --- a/deploy/opal_poll/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '3.5' -services: - - opal: - container_name: datashield_opal - restart: unless-stopped - image: docker.miracum.org/datashield_miracum/ds_opal:v0.10.1 - ports: - - "443:8443" - networks: - - opal_net - environment: - - OPAL_ADMINISTRATOR_PASSWORD=${OPAL_ADMIN_PASS} - - RSERVER_PORT_6312_TCP_ADDR=${R_SERVER_HOST} - - MONGODBHOST=${OPAL_MONGODB_HOST} - - INITTESTDATA=true - - DS_VERSION=${DS_VERSION:-6.0.0} - - DS_PRIVACY_LEVEL=$DS_PRIVACY_LEVEL - volumes: - - ${QP_DATA_DIR}/opal:/srv - - /etc/dsqp/auth:/auth - - /etc/dsqp/miracum_users:/miracum_users - - rserver: - container_name: datashield_rserver - restart: unless-stopped - image: obiba/opal-rserver:1.6 - networks: - - opal_net - - mongo: - container_name: datashield_mongo - restart: unless-stopped - image: mongo:4 - volumes: - - ${QP_DATA_DIR}/mongo_data:/data/db - networks: - - opal_net - - poll_server: - container_name: poll_server - restart: unless-stopped - image: docker.miracum.org/datashield_miracum/ds_poll:v0.9.1 - environment: - - POLL_QUEUE_SERVER=${POLL_QUEUE_SERVER} - - POLL_OPAL_SERVER=${POLL_OPAL_SERVER} - - POLL_THREADS=${POLL_THREADS} - - CHECK_SERVER_CERT=${CHECK_SERVER_CERT} - - LOG_LEVEL=${LOG_LEVEL} - entrypoint: bash -c "tail -f /dev/null" - networks: - - opal_net - volumes: - - ${QP_DATA_DIR}/poll_logging:/home/dspoll/ds_poll/logging - - /etc/dsqp/auth:/auth - -networks: - opal_net: - name: ds_opal_opal_net diff --git a/deploy/opal_poll/install.sh b/deploy/opal_poll/install.sh deleted file mode 100755 index ce93d14..0000000 --- a/deploy/opal_poll/install.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -source .env -QP_HOME_DIR=${QP_HOME_DIR:-"$PWD"} - - -if [[ $(which docker) && $(which docker-compose) ]]; then - echo "docker and docker compose already installed, versions are: " - docker -v - docker-compose -v -else - echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" - echo "aborting installation" - exit -fi - -printf "**** Creating directory /etc/dsqp for config files and copying config files to /etc/dsqp directory ...\n\n" -mkdir -p $QP_DATA_DIR -mkdir -p /etc/dsqp/auth -mkdir -p /etc/dsqp/miracum_users -mkdir -p /etc/dsqp/scripts -cp $QP_HOME_DIR/.env /etc/dsqp/.env -cp $QP_HOME_DIR/docker-compose.yml /etc/dsqp/docker-compose.yml -cp $QP_HOME_DIR/start.sh /etc/dsqp/start.sh -cp $QP_HOME_DIR/stop.sh /etc/dsqp/stop.sh -cp $QP_HOME_DIR/update.sh /etc/dsqp/update.sh - -cp -R $QP_HOME_DIR/auth/* /etc/dsqp/auth -cp -R $QP_HOME_DIR/miracum_users/* /etc/dsqp/miracum_users -cp -R $QP_HOME_DIR/scripts/* /etc/dsqp/scripts - - -cd /etc/dsqp - -./start.sh - -printf "\n the first time opal starts it takes a while to be ready as we are loading test data and configuring the servers for you, so please be patient\n" - -printf "\n - visit https://$OPAL_SERVER_IP:443 in your browser to access the opal server user interface \n" diff --git a/deploy/opal_poll/scripts/addCaCertificates.sh b/deploy/opal_poll/scripts/addCaCertificates.sh deleted file mode 100755 index 14e50bc..0000000 --- a/deploy/opal_poll/scripts/addCaCertificates.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - - -docker exec -it poll_server bash -c "cp /auth/ca_certs/* /usr/local/share/ca-certificates && update-ca-certificates" diff --git a/deploy/opal_poll/scripts/add_miracum_import_user.sh b/deploy/opal_poll/scripts/add_miracum_import_user.sh deleted file mode 100644 index fc48f3c..0000000 --- a/deploy/opal_poll/scripts/add_miracum_import_user.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -USER="miracum_import" -PW=$(pwgen -s 30 1) - - -docker exec --user=root datashield_opal bash -c "/miracum/init_miracum_import_user.sh $USER $PW" - -echo "Created Import User and written user information to /etc/dsqp/miracum_projects.config" -echo "Created User: $USER with password: $PW" >> /etc/dsqp/miracum_projects.config - diff --git a/deploy/opal_poll/scripts/add_miracum_projects.sh b/deploy/opal_poll/scripts/add_miracum_projects.sh deleted file mode 100644 index 67e9158..0000000 --- a/deploy/opal_poll/scripts/add_miracum_projects.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -source /etc/dsqp/.env -len=${#MIRACUM_PROJECTS[@]} - - -printf "Creating miracum projects - note: the miracum import user will get permission to administer the projects" - -for (( i=0; i<$len; i++ )); - do - - PROJECT=${MIRACUM_PROJECTS[$i]} - docker exec datashield_opal bash -c "/miracum/init_miracum_project.sh $PROJECT" - done diff --git a/deploy/opal_poll/scripts/add_miracum_users.sh b/deploy/opal_poll/scripts/add_miracum_users.sh deleted file mode 100644 index bd833f1..0000000 --- a/deploy/opal_poll/scripts/add_miracum_users.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -source /etc/dsqp/.env - -for user in ${MIRACUM_USERS[@]} -do - echo $user - CERT_FILE=/miracum_users/users/$user.pem - - if [ -f "$CERT_FILE" ]; then - docker exec datashield_opal bash -c "/miracum/create_miracum_user.sh $user $CERT_FILE CERT" - else - PW=$(pwgen -s 30 1) - docker exec datashield_opal bash -c "/miracum/create_miracum_user.sh $user $PW" - echo "Created new User $user and written user information to /etc/dsqp/miracum_projects.config" - echo "Created User: $user with password: $PW" >> /etc/dsqp/miracum_projects.config - fi - -done diff --git a/deploy/opal_poll/scripts/add_user_to_table.sh b/deploy/opal_poll/scripts/add_user_to_table.sh deleted file mode 100644 index d60f232..0000000 --- a/deploy/opal_poll/scripts/add_user_to_table.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -PROJECT=$1 -TABLE=$2 -USER=$3 - - -printf "Givign user $USER permission to access table $TABLE of project $PROJECT \n\n " - -docker exec datashield_opal bash -c "/miracum/add_user_to_table.sh $PROJECT $TABLE $USER" diff --git a/deploy/opal_poll/scripts/import_certificate.sh b/deploy/opal_poll/scripts/import_certificate.sh deleted file mode 100644 index a3d36b2..0000000 --- a/deploy/opal_poll/scripts/import_certificate.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - - -docker exec datashield_opal bash -c "cd /auth && chmod +x import_certificate.sh && ./import_certificate.sh" diff --git a/deploy/opal_poll/scripts/poll_admin.sh b/deploy/opal_poll/scripts/poll_admin.sh deleted file mode 100755 index e4a1ce7..0000000 --- a/deploy/opal_poll/scripts/poll_admin.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -COMMAND=$1 -QUEUE_SERVER=${2:-""} -OPAL_SERVER=${3:-"-o datashield_opal:8443"} -POLL_THREADS=${4:-""} -CHECK_SERVER_CERT=${5:-""} - -case "$COMMAND" in - start ) - echo "Starting poll with queue server: $QUEUE_SERVER, opal server: $OPAL_SERVER and number of poll threads: $POLL_THREADS" - docker exec -dit poll_server bash -c "/home/dspoll/poll_admin.sh start '$QUEUE_SERVER' '$OPAL_SERVER' '$POLL_THREADS' '$CHECK_SERVER_CERT'" - ;; - - stop ) - echo "Stopping poll ..." - docker exec -it poll_server bash -c "/home/dspoll/poll_admin.sh stop" - ;; - - status ) - docker exec -it poll_server bash -c "/home/dspoll/poll_admin.sh status" - ;; - - * ) - echo $"Usage: $0 {start|stop|status|}" - exit 1 - -esac diff --git a/deploy/opal_poll/update.sh b/deploy/opal_poll/update.sh deleted file mode 100755 index 76757f5..0000000 --- a/deploy/opal_poll/update.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -source .env -while getopts v:d: option -do -case "${option}" -in -v) VERSION=${OPTARG};; -d) QP_HOME_DIR=${OPTARG};; - -esac -done - - -if [ -n "$VERSION" ]; then - wget -N "https://gitlab.miracum.org/miracum/uc2/datashield/ds_develop/raw/$VERSION/deploy/opal_poll/docker-compose.yml" -elif [ -n "$QP_HOME_DIR" ]; then - printf "****\n no version specified updating with data from this folder \n" - printf "****\n note that config and data files will be kept - if config files have changed you have to transfer them manually\n" - printf "**** Creating directories and copying files...\n\n" - - mkdir -p $QP_DATA_DIR - mkdir -p /etc/dsqp/miracum_users - mkdir -p /etc/dsqp/scripts - cp $QP_HOME_DIR/docker-compose.yml /etc/dsqp/docker-compose.yml - cp $QP_HOME_DIR/start.sh /etc/dsqp/start.sh - cp $QP_HOME_DIR/stop.sh /etc/dsqp/stop.sh - cp $QP_HOME_DIR/update.sh /etc/dsqp/update.sh - cp -R $QP_HOME_DIR/scripts/* /etc/dsqp/scripts - -else - printf "\n no version and no install dir given => not doing anything" - exit - -fi - -cd /etc/dsqp - -docker-compose down -docker-compose up -d - -printf "\n - visit your server IP or domain + port 8787 in your browser to access the analysis client \n" - - -printf "\n the first time opal starts it takes a while to be ready as we are loading test data and configuring the servers for you, so please be patient\n" - -printf "\n - visit $OPAL_SERVER_IP:8443 in your browser to access the poll server user interface \n" -printf "\n - visit https://$OPAL_SERVER_IP:443 in your browser to access the opal server user interface \n" diff --git a/deploy/poll/.env.default b/deploy/poll/.env.default new file mode 100755 index 0000000..fba6f8f --- /dev/null +++ b/deploy/poll/.env.default @@ -0,0 +1,13 @@ +# this file lets you install opal and the poll service + +### add your configuration information here + +QP_DOCKER_PROJECT=ds-qp + +# e.g. 123.12.12.12:443 , queue server host and port(usually 443) +POLL_QUEUE_SERVER= +# e.g. '123.12.12.12:443' , opal server host and port, default is set to datashield_opal:8443, which is correct for a pure docker setup +POLL_OPAL_SERVER= +# e.g. 5 number of poll threads default 2 +POLL_THREADS=2 +LOG_LEVEL=20 \ No newline at end of file diff --git a/deploy/poll/docker-compose.yml b/deploy/poll/docker-compose.yml new file mode 100644 index 0000000..bdaa227 --- /dev/null +++ b/deploy/poll/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.5' +services: + + poll: + restart: unless-stopped + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_poll:sha256-c4d250b7a6e81929283b281e20832e958ea590294a1ebf932a799abe70e6329b.sig + restart: unless-stopped + environment: + - POLL_QUEUE_SERVER=${POLL_QUEUE_SERVER} + - POLL_OPAL_SERVER=${POLL_OPAL_SERVER:-opal:8443} + - POLL_THREADS=${POLL_THREADS:-2} + - OWN_CERT_CA=${OWN_CERT_CA} + - LOG_LEVEL=${LOG_LEVEL:-20} \ No newline at end of file diff --git a/deploy/opal_poll/start.sh b/deploy/poll/start.sh similarity index 93% rename from deploy/opal_poll/start.sh rename to deploy/poll/start.sh index 3b5752a..77262ac 100755 --- a/deploy/opal_poll/start.sh +++ b/deploy/poll/start.sh @@ -11,9 +11,9 @@ if [[ ! $QUEUE_AVAIL == "queue is still alive " ]];then printf "\n The queue is not available, the following curl ping request failed: curl -k https://$POLL_QUEUE_SERVER/?ping=true\n" printf "\n Exiting now - reconfigure your variable POLL_QUEUE_SERVER in your .env file in the folder /etc/dsqp and execute the ./start.sh again \n" exit -fi +fi printf "######################\nInitialising Opal and installing Datashield and RServer ...\n######################\n\n" -docker-compose up -d +docker-compose up -p $QP_DOCKER_PROJECT -d diff --git a/deploy/poll/stop.sh b/deploy/poll/stop.sh new file mode 100644 index 0000000..7495a72 --- /dev/null +++ b/deploy/poll/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +printf "######################\Stopping Poll ...\n######################\n\n" + +docker-compose -p $QP_DOCKER_PROJECT stop \ No newline at end of file diff --git a/deploy/queue/.env b/deploy/queue/.env.default similarity index 97% rename from deploy/queue/.env rename to deploy/queue/.env.default index e93846c..a0c5496 100755 --- a/deploy/queue/.env +++ b/deploy/queue/.env.default @@ -24,6 +24,6 @@ TIMEOUT_QUEUE_AND_POLL=100:604800 # to function. ## -ALLOWED_IPS=140.0.0.1,140.0.0.1 +ALLOWED_IPS= ### END CONFIGURATIONS ### \ No newline at end of file diff --git a/deploy/queue/nginx/nginx.conf b/deploy/queue/auth/nginx.conf similarity index 99% rename from deploy/queue/nginx/nginx.conf rename to deploy/queue/auth/nginx.conf index ba5f5e1..a97d086 100644 --- a/deploy/queue/nginx/nginx.conf +++ b/deploy/queue/auth/nginx.conf @@ -63,5 +63,6 @@ gzip_disable "MSIE [1-6]\."; include /etc/nginx/conf.d/queue_allow_ips.conf; proxy_pass https://queue_server:8443/; } + } -} \ No newline at end of file +} diff --git a/deploy/queue/docker-compose.yml b/deploy/queue/docker-compose.yml index fac6b2e..f70cde2 100644 --- a/deploy/queue/docker-compose.yml +++ b/deploy/queue/docker-compose.yml @@ -1,10 +1,9 @@ version: '2.0' services: - nginx_queue: - container_name: nginx_queue + queue_nginx: restart: unless-stopped - image: docker.miracum.org/datashield_miracum/nginx_queue:v0.8.4 + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue_nginx:sha256-a9e70772c43a4ceb3e447293f01a6ac42d133f7313a5e2c982ab8ecca1181b64.sig ports: - "443:8443" networks: @@ -12,24 +11,19 @@ services: environment: - ALLOWED_IPS=${ALLOWED_IPS} volumes: - - /etc/dsqp/nginx/queuecert.pem:/etc/nginx/conf.d/queuecert.pem - - /etc/dsqp/nginx/queuekey.pem:/etc/nginx/conf.d/queuekey.pem - - /etc/dsqp/nginx/dhparam.pem:/etc/nginx/dhparam.pem - - /etc/dsqp/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./auth/key.pem:/etc/nginx/conf.d/queuekey.pem + - ./auth/cert.pem:/etc/nginx/conf.d/queuecert.pem + - ./auth/dhparam.pem:/etc/nginx/dhparam.pem + - /auth/nginx.conf:/etc/nginx/nginx.conf:ro queue_server: - container_name: queue_server + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue:sha256-f5bcb75b264aee2e47ba7e2bb9f931e5d8acfe09daf5ad7d7736de97c0c003de.sig restart: unless-stopped - image: docker.miracum.org/datashield_miracum/ds_queue:v0.8.4 volumes: - ${QP_DATA_DIR}/queue_logging:/home/dsqueue/ds_queue/logging - - /etc/dsqp/auth/queue.pem:/root/ds_queue/cert/do_cert/queue.pem + - ./auth/key.pem:/home/dsqueue/ds_queue/cert/queuekey.pem + - ./auth/cert.pem:/home/dsqueue/ds_queue/cert/queuecert.pem environment: - TIMEOUT_QUEUE_AND_POLL=${TIMEOUT_QUEUE_AND_POLL} - ALLOWED_IPS=${ALLOWED_IPS} - LOG_LEVEL=${LOG_LEVEL} - networks: - - queue_net - -networks: - queue_net: diff --git a/deploy/queue/install.sh b/deploy/queue/install.sh deleted file mode 100755 index 8237e10..0000000 --- a/deploy/queue/install.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# this scripts installs the queue server for you -source .env -QP_HOME_DIR=${QP_HOME_DIR:-"$PWD"} - -if [[ $(which docker) && $(which docker-compose) ]]; then - echo "docker and docker compose already installed, versions are: " - docker -v - docker-compose -v -else - echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" - echo "aborting installation" - exit -fi - - -printf "**** Creating directory /etc/dsqp for config files and copying unzipped config files to /etc/dsqp directory ...\n\n" -mkdir -p /etc/dsqp/auth -mkdir /etc/dsqp/nginx -mkdir -p $QP_DATA_DIR -cp $QP_HOME_DIR/.env /etc/dsqp/.env -cp $QP_HOME_DIR/docker-compose.yml /etc/dsqp/docker-compose.yml -cp $QP_HOME_DIR/start.sh /etc/dsqp/start.sh -cp $QP_HOME_DIR/stop.sh /etc/dsqp/stop.sh -cp $QP_HOME_DIR/update.sh /etc/dsqp/update.sh -cp -R $QP_HOME_DIR/auth/queue.pem /etc/dsqp/auth/queue.pem -cp -R $QP_HOME_DIR/nginx/* /etc/dsqp/nginx - -printf "**** removing config files from home repository $QP_HOME_DIR...\n\n" -rm $QP_HOME_DIR/.env -rm -rf $QP_HOME_DIR/auth -rm -rf $QP_HOME_DIR/nginx - -FILE=/etc/dsqp/nginx/dhparam.pem -if [ ! -f "$FILE" ]; then - echo "Creating longer Diffie-Hellman Prime for extra security... this may take a while \n\n" - openssl dhparam -out /etc/dsqp/nginx/dhparam.pem 4096 -fi - -CERT_FILE=/etc/dsqp/auth/queuecert.pem -KEY_FILE=/etc/dsqp/auth/queuekey.pem -if [ ! -f "$CERT_FILE" ] || [ ! -f "$KEY_FILE" ]; then - echo "ERROR missing certificates for nginx proxy" - echo "please create a $CERT_FILE and $KEY_FILE accordingly" - echo "aborting installation" - exit -fi - -cd /etc/dsqp - -docker-compose up -d - -printf "\n - check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file diff --git a/deploy/queue/nginx/mime.types b/deploy/queue/nginx/mime.types deleted file mode 100644 index 62bd4b6..0000000 --- a/deploy/queue/nginx/mime.types +++ /dev/null @@ -1,48 +0,0 @@ -types { - text/html html htm shtml; - text/css css; - text/xml xml rss; - image/gif gif; - image/jpeg jpeg jpg; - application/x-javascript js; - text/plain txt; - text/x-component htc; - text/mathml mml; - image/png png; - image/x-icon ico; - image/x-jng jng; - image/vnd.wap.wbmp wbmp; - application/java-archive jar war ear; - application/mac-binhex40 hqx; - application/pdf pdf; - application/x-cocoa cco; - application/x-java-archive-diff jardiff; - application/x-java-jnlp-file jnlp; - application/x-makeself run; - application/x-perl pl pm; - application/x-pilot prc pdb; - application/x-rar-compressed rar; - application/x-redhat-package-manager rpm; - application/x-sea sea; - application/x-shockwave-flash swf; - application/x-stuffit sit; - application/x-tcl tcl tk; - application/x-x509-ca-cert der pem crt; - application/x-xpinstall xpi; - application/zip zip; - application/octet-stream deb; - application/octet-stream bin exe dll; - application/octet-stream dmg; - application/octet-stream eot; - application/octet-stream iso img; - application/octet-stream msi msp msm; - audio/mpeg mp3; - audio/x-realaudio ra; - video/mpeg mpeg mpg; - video/quicktime mov; - video/x-flv flv; - video/x-msvideo avi; - video/x-ms-wmv wmv; - video/x-ms-asf asx asf; - video/x-mng mng; -} \ No newline at end of file diff --git a/deploy/queue/nginx/q_nginx_start.sh b/deploy/queue/nginx/q_nginx_start.sh deleted file mode 100755 index 356c1b3..0000000 --- a/deploy/queue/nginx/q_nginx_start.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -ALLOWED_IPS=${ALLOWED_IPS:-""} - -echo $ALLOWED_IPS - -ips=$(echo $ALLOWED_IPS | tr ",'" "\n") - -printf "" > /etc/nginx/conf.d/queue_allow_ips.conf - -for ip in $ips -do - echo "allow $ip;" >> /etc/nginx/conf.d/queue_allow_ips.conf -done - -if [[ -n $ALLOWED_IPS ]];then - echo "deny all;" >> /etc/nginx/conf.d/queue_allow_ips.conf -fi - -#starting nginx -nginx -g 'daemon off;' diff --git a/deploy/queue/start.sh b/deploy/queue/start.sh new file mode 100755 index 0000000..8f51484 --- /dev/null +++ b/deploy/queue/start.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [[ $(which docker) && $(which docker-compose) ]]; then + echo "docker and docker compose already installed, versions are: " + docker -v + docker-compose -v +else + echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" + echo "aborting installation" + exit +fi + +FILE=auth/dhparam.pem +if [ ! -f "$FILE" ]; then + echo "Creating longer Diffie-Hellman Prime for extra security... this may take a while \n\n" + openssl dhparam -out /etc/dsqp/nginx/dhparam.pem 4096 +fi + +KEY_FILE=auth/key.pem +CERT_FILE=auth/cert.pem +if [ ! -f "$KEY_FILE" ] || [ ! -f "$CERT_FILE" ]; then + echo "ERROR missing certificates for nginx proxy" + echo "please create a $CERT_FILE and $KEY_FILE accordingly" + echo "aborting installation" + exit +fi + +docker-compose up -d + +printf "\n - Check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file diff --git a/deploy/queue/update.sh b/deploy/queue/update.sh deleted file mode 100755 index 5e2444d..0000000 --- a/deploy/queue/update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -source .env - -while getopts v:d: option -do -case "${option}" -in -v) VERSION=${OPTARG};; -d) QP_HOME_DIR=${OPTARG};; - -esac -done - -if [ -n "$VERSION" ]; then - wget -N "https://gitlab.miracum.org/miracum/uc2/datashield/ds_develop/raw/$VERSION/deploy/queue/docker-compose.yml" -elif [ -n "$QP_HOME_DIR" ]; then - printf "****\n no version specified updating with data from this folder \n" - printf "****\n note that config and data files will be kept - if config files have changed you have to transfer them manually\n" - printf "**** Creating directories and copying files...\n\n" - - mkdir -p $QP_DATA_DIR - cp $QP_HOME_DIR/docker-compose.yml /etc/dsqp/docker-compose.yml - cp $QP_HOME_DIR/start.sh /etc/dsqp/start.sh - cp $QP_HOME_DIR/stop.sh /etc/dsqp/stop.sh - cp $QP_HOME_DIR/update.sh /etc/dsqp/update.sh - -else - printf "\n no version and no install dir given => not doing anything" - exit - -fi - -cd /etc/dsqp - -docker-compose down -docker-compose up -d - -printf "\n - check if queue is running by typing 'docker ps' into the command line \n" diff --git a/deploy/test.py b/deploy/test.py deleted file mode 100644 index 44a251a..0000000 --- a/deploy/test.py +++ /dev/null @@ -1,5 +0,0 @@ -import base64 - -data= "botty:h}2n(!~VqL$:zAFt" -encodedBytes = base64.b64encode(data.encode("utf-8")) -print(encodedBytes) \ No newline at end of file diff --git a/deploy/test/.env b/deploy/test/.env deleted file mode 100755 index 4ff655b..0000000 --- a/deploy/test/.env +++ /dev/null @@ -1,10 +0,0 @@ -# version of qp and ds system - e.g. v0.0.1 - default is empty => master = newest -QP_VERSION_TAG=v0.7.5 -# registry host:port e.g. 'testregistry.de:5043' -QP_DOCKER_REGISTRY_PREFIX=docker.miracum.org/datashield_miracum -# user e.g. 'test' -QP_DOCKER_REGISTRY_USER= -# password, e.g. 'test123!' -QP_DOCKER_REGISTRY_PW= -# change queue host to your host -QP_QUEUE_HOST=https://nginx_queue:443 \ No newline at end of file diff --git a/deploy/test/.env.default b/deploy/test/.env.default new file mode 100755 index 0000000..c883f82 --- /dev/null +++ b/deploy/test/.env.default @@ -0,0 +1,2 @@ +# version of qp and ds system - e.g. v0.0.1 - default is empty => master = newest +QP_QUEUE_HOST=https://nginx_queue:443 \ No newline at end of file diff --git a/deploy/test/docker-compose.yml b/deploy/test/docker-compose.yml new file mode 100644 index 0000000..a64508d --- /dev/null +++ b/deploy/test/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.2' +services: + ds_test: + #image: TODO + volumes: + - ./testscript:/testscript + environment: + - QUEUE_HOST=${QUEUE_HOST} diff --git a/deploy/test/ds_test.sh b/deploy/test/ds_test.sh old mode 100755 new mode 100644 diff --git a/ds_queue/nginx/nginx.conf b/ds_queue/nginx/nginx.conf index 2d96be6..77a7fa3 100644 --- a/ds_queue/nginx/nginx.conf +++ b/ds_queue/nginx/nginx.conf @@ -39,5 +39,10 @@ gzip_disable "MSIE [1-6]\."; include /etc/nginx/conf.d/queue_allow_ips.conf; proxy_pass https://queue_server:443/; } + + location /qprequest { + deny all; + return + } } } \ No newline at end of file From d98d4a11ea519ce95607220e25b4665d54e6c040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gru=CC=88ndner?= Date: Fri, 1 Jul 2022 15:39:27 +0200 Subject: [PATCH 2/5] fix: remove unnecessary env vars and fix docker-compose call in start.sh --- deploy/opal/.env.default | 5 ++--- deploy/opal/start.sh | 2 +- deploy/poll/start.sh | 2 +- deploy/queue/start.sh | 2 +- deploy/test/.env.default | 1 + 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/opal/.env.default b/deploy/opal/.env.default index 6a1c8b7..86db3b3 100755 --- a/deploy/opal/.env.default +++ b/deploy/opal/.env.default @@ -2,11 +2,10 @@ ### add your configuration information here QP_DOCKER_PROJECT=ds-qp -QP_HOME_DIR= -QP_DATA_DIR=/opt/dsqp + # please change this password to a save password OPAL_ADMIN_PASS=password OPAL_ROCKS_HOSTS= OPAL_MONGODB_HOST= -DS_VERSION=6.0.0 +DS_VERSION=6.1.1 DS_PRIVACY_LEVEL=5 \ No newline at end of file diff --git a/deploy/opal/start.sh b/deploy/opal/start.sh index 7afc6a1..d4d197e 100755 --- a/deploy/opal/start.sh +++ b/deploy/opal/start.sh @@ -21,6 +21,6 @@ if [ ! -f "$KEY_FILE" ] || [ ! -f "$CERT_FILE" ]; then exit fi -docker-compose up -p $QP_DOCKER_PROJECT -d +docker-compose -p $QP_DOCKER_PROJECT up -d printf "\n - Check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file diff --git a/deploy/poll/start.sh b/deploy/poll/start.sh index 77262ac..8f49cc7 100755 --- a/deploy/poll/start.sh +++ b/deploy/poll/start.sh @@ -16,4 +16,4 @@ fi printf "######################\nInitialising Opal and installing Datashield and RServer ...\n######################\n\n" -docker-compose up -p $QP_DOCKER_PROJECT -d +docker-compose -p $QP_DOCKER_PROJECT up -d diff --git a/deploy/queue/start.sh b/deploy/queue/start.sh index 8f51484..364243a 100755 --- a/deploy/queue/start.sh +++ b/deploy/queue/start.sh @@ -13,7 +13,7 @@ fi FILE=auth/dhparam.pem if [ ! -f "$FILE" ]; then echo "Creating longer Diffie-Hellman Prime for extra security... this may take a while \n\n" - openssl dhparam -out /etc/dsqp/nginx/dhparam.pem 4096 + openssl dhparam -out auth/dhparam.pem 4096 fi KEY_FILE=auth/key.pem diff --git a/deploy/test/.env.default b/deploy/test/.env.default index c883f82..cbde571 100755 --- a/deploy/test/.env.default +++ b/deploy/test/.env.default @@ -1,2 +1,3 @@ # version of qp and ds system - e.g. v0.0.1 - default is empty => master = newest + QP_QUEUE_HOST=https://nginx_queue:443 \ No newline at end of file From ce32098c03d9a81352272f397b7857034123b46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gru=CC=88ndner?= Date: Fri, 1 Jul 2022 17:18:06 +0200 Subject: [PATCH 3/5] change to correct images --- deploy/opal/docker-compose.yml | 2 +- deploy/opal/stop.sh | 1 + deploy/poll/docker-compose.yml | 2 +- deploy/queue/docker-compose.yml | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/opal/docker-compose.yml b/deploy/opal/docker-compose.yml index 1e4dc09..147bb99 100644 --- a/deploy/opal/docker-compose.yml +++ b/deploy/opal/docker-compose.yml @@ -1,6 +1,6 @@ services: opal: - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_opal:sha256-5de21239f4c8987728c840c2de8ba8d76d386f59d1ec9d9a3626ac21b25b40b6.sig + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_opal:v0.0.1 restart: unless-stopped ports: - "8443:8443" diff --git a/deploy/opal/stop.sh b/deploy/opal/stop.sh index afc4079..13b5d35 100644 --- a/deploy/opal/stop.sh +++ b/deploy/opal/stop.sh @@ -1,4 +1,5 @@ #!/bin/bash +source .env printf "######################\Stopping Opal, Datashield and RServer ...\n######################\n\n" docker-compose -p $QP_DOCKER_PROJECT stop \ No newline at end of file diff --git a/deploy/poll/docker-compose.yml b/deploy/poll/docker-compose.yml index bdaa227..a7d27e2 100644 --- a/deploy/poll/docker-compose.yml +++ b/deploy/poll/docker-compose.yml @@ -3,7 +3,7 @@ services: poll: restart: unless-stopped - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_poll:sha256-c4d250b7a6e81929283b281e20832e958ea590294a1ebf932a799abe70e6329b.sig + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_poll:v0.0.1 restart: unless-stopped environment: - POLL_QUEUE_SERVER=${POLL_QUEUE_SERVER} diff --git a/deploy/queue/docker-compose.yml b/deploy/queue/docker-compose.yml index f70cde2..7f246d1 100644 --- a/deploy/queue/docker-compose.yml +++ b/deploy/queue/docker-compose.yml @@ -3,7 +3,7 @@ version: '2.0' services: queue_nginx: restart: unless-stopped - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue_nginx:sha256-a9e70772c43a4ceb3e447293f01a6ac42d133f7313a5e2c982ab8ecca1181b64.sig + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue:v0.0.1 ports: - "443:8443" networks: @@ -17,7 +17,7 @@ services: - /auth/nginx.conf:/etc/nginx/nginx.conf:ro queue_server: - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue:sha256-f5bcb75b264aee2e47ba7e2bb9f931e5d8acfe09daf5ad7d7736de97c0c003de.sig + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue_nginx:v0.0.1 restart: unless-stopped volumes: - ${QP_DATA_DIR}/queue_logging:/home/dsqueue/ds_queue/logging From 6e178126bb0d713713d7f033a6642b6c6278b254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gru=CC=88ndner?= Date: Fri, 1 Jul 2022 21:26:38 +0200 Subject: [PATCH 4/5] fix nginx config, source .env in poll scripts and add cmake to simple client Dockerfile --- deploy/poll/start.sh | 2 +- deploy/poll/stop.sh | 2 ++ deploy/queue/.env.default | 2 ++ deploy/queue/auth/nginx.conf | 2 +- deploy/queue/docker-compose.yml | 6 ++--- deploy/queue/start.sh | 29 +----------------------- deploy/queue/stop.sh | 30 +++++++++++++++++++++++++ deploy/simple_client/docker-compose.yml | 7 ++++++ ds_simple_client/Dockerfile | 3 ++- 9 files changed, 48 insertions(+), 35 deletions(-) create mode 100755 deploy/queue/stop.sh create mode 100644 deploy/simple_client/docker-compose.yml diff --git a/deploy/poll/start.sh b/deploy/poll/start.sh index 8f49cc7..56a7a59 100755 --- a/deploy/poll/start.sh +++ b/deploy/poll/start.sh @@ -14,6 +14,6 @@ exit fi -printf "######################\nInitialising Opal and installing Datashield and RServer ...\n######################\n\n" +printf "######################\n Starting Polling ...\n######################\n\n" docker-compose -p $QP_DOCKER_PROJECT up -d diff --git a/deploy/poll/stop.sh b/deploy/poll/stop.sh index 7495a72..bb3f8e9 100644 --- a/deploy/poll/stop.sh +++ b/deploy/poll/stop.sh @@ -1,5 +1,7 @@ #!/bin/bash +source .env + printf "######################\Stopping Poll ...\n######################\n\n" docker-compose -p $QP_DOCKER_PROJECT stop \ No newline at end of file diff --git a/deploy/queue/.env.default b/deploy/queue/.env.default index a0c5496..f29b1e9 100755 --- a/deploy/queue/.env.default +++ b/deploy/queue/.env.default @@ -1,6 +1,8 @@ ### BEGIN CONFIGURATIONS ### # uncomment variables to set configuration +QP_DOCKER_PROJECT=ds-qp + QP_HOME_DIR= QP_DATA_DIR=/opt/dsqp LOG_LEVEL=20 diff --git a/deploy/queue/auth/nginx.conf b/deploy/queue/auth/nginx.conf index a97d086..3ca245e 100644 --- a/deploy/queue/auth/nginx.conf +++ b/deploy/queue/auth/nginx.conf @@ -61,7 +61,7 @@ gzip_disable "MSIE [1-6]\."; location / { include /etc/nginx/conf.d/queue_allow_ips.conf; - proxy_pass https://queue_server:8443/; + proxy_pass https://queue_server:443/; } } diff --git a/deploy/queue/docker-compose.yml b/deploy/queue/docker-compose.yml index 7f246d1..6646ced 100644 --- a/deploy/queue/docker-compose.yml +++ b/deploy/queue/docker-compose.yml @@ -3,11 +3,9 @@ version: '2.0' services: queue_nginx: restart: unless-stopped - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue:v0.0.1 + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue_nginx:v0.0.1 ports: - "443:8443" - networks: - - queue_net environment: - ALLOWED_IPS=${ALLOWED_IPS} volumes: @@ -17,7 +15,7 @@ services: - /auth/nginx.conf:/etc/nginx/nginx.conf:ro queue_server: - image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue_nginx:v0.0.1 + image: ghcr.io/medizininformatik-initiative/distributed-analysis_datashield-qp/ds_queue:v0.0.1 restart: unless-stopped volumes: - ${QP_DATA_DIR}/queue_logging:/home/dsqueue/ds_queue/logging diff --git a/deploy/queue/start.sh b/deploy/queue/start.sh index 364243a..1bb196b 100755 --- a/deploy/queue/start.sh +++ b/deploy/queue/start.sh @@ -1,30 +1,3 @@ #!/bin/bash -if [[ $(which docker) && $(which docker-compose) ]]; then - echo "docker and docker compose already installed, versions are: " - docker -v - docker-compose -v -else - echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" - echo "aborting installation" - exit -fi - -FILE=auth/dhparam.pem -if [ ! -f "$FILE" ]; then - echo "Creating longer Diffie-Hellman Prime for extra security... this may take a while \n\n" - openssl dhparam -out auth/dhparam.pem 4096 -fi - -KEY_FILE=auth/key.pem -CERT_FILE=auth/cert.pem -if [ ! -f "$KEY_FILE" ] || [ ! -f "$CERT_FILE" ]; then - echo "ERROR missing certificates for nginx proxy" - echo "please create a $CERT_FILE and $KEY_FILE accordingly" - echo "aborting installation" - exit -fi - -docker-compose up -d - -printf "\n - Check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file +docker-compose stop \ No newline at end of file diff --git a/deploy/queue/stop.sh b/deploy/queue/stop.sh new file mode 100755 index 0000000..364243a --- /dev/null +++ b/deploy/queue/stop.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [[ $(which docker) && $(which docker-compose) ]]; then + echo "docker and docker compose already installed, versions are: " + docker -v + docker-compose -v +else + echo "ERROR docker and/or docker-compose not installed, please install docker and docker compose" + echo "aborting installation" + exit +fi + +FILE=auth/dhparam.pem +if [ ! -f "$FILE" ]; then + echo "Creating longer Diffie-Hellman Prime for extra security... this may take a while \n\n" + openssl dhparam -out auth/dhparam.pem 4096 +fi + +KEY_FILE=auth/key.pem +CERT_FILE=auth/cert.pem +if [ ! -f "$KEY_FILE" ] || [ ! -f "$CERT_FILE" ]; then + echo "ERROR missing certificates for nginx proxy" + echo "please create a $CERT_FILE and $KEY_FILE accordingly" + echo "aborting installation" + exit +fi + +docker-compose up -d + +printf "\n - Check if queue is running by typing 'docker ps' into the command line \n" \ No newline at end of file diff --git a/deploy/simple_client/docker-compose.yml b/deploy/simple_client/docker-compose.yml new file mode 100644 index 0000000..3ba23c4 --- /dev/null +++ b/deploy/simple_client/docker-compose.yml @@ -0,0 +1,7 @@ +services: + ds_simple_client: + image: # TODO + tty: true + volumes: + - ../cert/ca_cert/ds_develop_ca.crt:/usr/local/share/ca-certificates/ds_develop_ca.crt + - ./testscripts:/testscripts diff --git a/ds_simple_client/Dockerfile b/ds_simple_client/Dockerfile index 821572e..aa93931 100644 --- a/ds_simple_client/Dockerfile +++ b/ds_simple_client/Dockerfile @@ -13,7 +13,8 @@ RUN apt-get update && \ apt-get install -y r-base r-base-dev && \ apt-get install -y libcurl4-openssl-dev libssl-dev && \ apt-get install -y libxml2-utils && \ -apt-get install -y libxml2-dev +apt-get install -y libxml2-dev && \ +apt-get install -y cmake # install opal packages RUN R -e "install.packages('DSOpal', repos=c(getOption('repos'), 'https://cran.obiba.org'))" && \ From 05c859803e86fe378760795581702b990fe3ea38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Gru=CC=88ndner?= Date: Wed, 6 Jul 2022 11:52:13 +0200 Subject: [PATCH 5/5] Update readme to new deploy version --- .gitignore | 3 +- README.md | 2 ++ deploy/README.md | 48 +++++++++++++++++++++++++++++++ deploy/init-env-files.sh | 13 +++++++++ deploy/opal/README.md | 62 ---------------------------------------- 5 files changed, 65 insertions(+), 63 deletions(-) create mode 100644 deploy/README.md create mode 100644 deploy/init-env-files.sh delete mode 100644 deploy/opal/README.md diff --git a/.gitignore b/.gitignore index 2cb7778..6ea4053 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ queue.config /cert/do_cert/* !/cert/do_cert/.gitkeep /cert/ca_cert/* -!/cert/ca_cert/.gitkeep \ No newline at end of file +!/cert/ca_cert/.gitkeep +/deploy/*/.env \ No newline at end of file diff --git a/README.md b/README.md index 48c04a3..934dd32 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,5 @@ This is a repository which provides a combination of docker packaged components ## Deployment Inside the deploy folder we provide the newest installation packages, which allow one to install the newest version of the DataSHIELD infrastructure. + +It also includes a README which explains the installation process. diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..ccfec7b --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,48 @@ +# Dockerized deployment of qp together with DataSHIELD + +## Prerequisites + +- Install Docker and docker-compose in the current version +- Including the DataSHIELD Opal server, the qp extension consists of four different components (see publication). +- Checkout this repository to your machine. + +### Server certificates and keys +You will need a key and a certificate for each of three of the Docker containers (queue server, NGINX for queue server, Opal server). These can be either regular trusted keys or keys trusted by a newly created Certificate Authority (CA), thus self-singed ones. To create your own CA and own keys you can use the scripts in the folder `cert`. Self-signed or regular, the keys and certificates need to be placed in the folder `cert/do_cert`. If your own CA is used, please ensure that this CA is added to the certificate store of the docker container. + +Note that self-signed certificates should only be used for develop. + +## Deployment of different components + +### Queue server and NGINX + +1. Change to the folder `deploy` of this repository +2. Execute the `init-env-files.sh` +3 Change to the folder `queue` of this repository +4. Edit the file `.env` and set the parameter `ALLOWED_IPS` to allowed IPs and subnets separated by a comma. The parameter should contain all allowed client IPs as well as the IPs of the poll server. +5. Add your certificates to the auth folder as `key.pem` and `cert.pem` files. +6. Start the queue server and the nginx using the `start.sh` script of the `queue` folder + +### Poll server +1. Change to the folder `deploy` of this repository. +2. Execute the `init-env-files.sh`. +3 Change to the folder `poll` of this repository. +4. Set the parameters `POLL_QUEUE_SERVER` and `POLL_OPAL_SERVER` in the file `.env` to the domain and ports of the queue server and the opal server. +5. Add your certificates to the auth folder as `key.pem` and `cert.pem` files. +6. Start the poll application using the `start.sh` script of the `poll` folder. + +### Opal server +1. Change to the folder `deploy` of this repository. +2. Execute the `init-env-files.sh`. +3 Change to the folder `opal` of this repository. +4. Edit the file `.env`: Set the parameter `OPAL_ADMIN_PASS` and choose a safe password. +5. Add your certificates to the auth folder as `key.pem` and `cert.pem` files. +6. Start the opal server and the nginx using the `start.sh` script of the `opal` folder. +7. To add a certificate to OPAL its easiest to open the ADMIN console and login as admin. Then click `General Settings` > `Set Key Pair` > `Import Key Pair` and paste your key and certificate in the respective fields and click `save`. +8. After adding your certificate for it to take effect you need to restart your opal server by first executing the `stop.sh` followed by the `start.sh` of the `opal` folder. + +## Troubleshooting +### Error reading certificates +The keys must be readable to the users of the Docker containers, which are not always root. In the containers' logs you may find messages indicating that the keys files are not readable. To enable reading the keys, you can make them readable with `chmod o+r `. However, in this case everybody on your system can access the key files. In order to restrict access you can change the owner of the file in the host system to the id of the running user in the Docker container, even if that user does not exists on the host system. For instance, execute `sudo chown 101:101 queuenginx.key` to allow the NGINX key to be read by the container. + +### Error writing queue_allow_ips.conf file +The NGINX container creates a file queue_allow_ips.conf at the beginning, which is stored in the folder `deploy/ds_queue/nginx` on your host system. If the Docker logs of the NGINX container put out a message like `/etc/nginx/conf.d/queue_allow_ips.conf: Permission denied` you can try to delete the file and restart the container. Also check whether the NGINX container can write to the folder `deploy/ds_queue/nginx`. diff --git a/deploy/init-env-files.sh b/deploy/init-env-files.sh new file mode 100644 index 0000000..a9865bb --- /dev/null +++ b/deploy/init-env-files.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +envfiles=( "opal/.env" "poll/.env" "queue/.env") + +for file in "${envfiles[@]}" +do + if [[ -f "$file" ]]; then + printf ".env file $file already exists - not copying default env \n" + printf "Please check if your current env file $file is missing any params from the $file.default file and copy them as appropriate\n" + else + cp "$file.default" $file + fi +done \ No newline at end of file diff --git a/deploy/opal/README.md b/deploy/opal/README.md deleted file mode 100644 index 4c56fc1..0000000 --- a/deploy/opal/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# DS Q-P + OPAL installation guide - - -## Install your poll thread, opal server and poll monitor server on a server - -The Q-P datashield opal application comes packed in docker images, which are automatically donwloaded from your registry of choice. - -*Please note that in order to run your poll thread you require a datashield queue server to connect to, so make sure u have installed this before you continue.* - -### To install the system: - -*open the opal_poll.config file and change the configuration information* to your server requirements. - -then execute `./install_prod_opal_poll.sh` in this repo. - -Thats it, your poll server, opal and datashield server is now installed. - -Please be aware that the server might take a couple of minutes to be available as the startup script automatically installs test data and updates the opal server. - - -### The config file explained - -Your installation package will be already configured to download the correct version from our docker registry. - -The most important parameters you will have to change are: -**OPAL_SERVER_IP**,**OPAL_ADMIN_PASS**,**POLL_QUEUE_SERVER**, the rest can stay as is. - -- export **OPAL_SERVER_IP**='127.0.0.1' # Ip address of this server (the server which hosts the opal and poll thread) - default is localhost -- export **OPAL_ADMIN_PASS**='password' # password for the opal server - it is important for this password to be a save password, we suggest using for example `pwgen -sy 20 1` on your command line to generate a safe password -- export **R_SERVER_HOST**='datashield_rserver' # host of an r server used for the opal processing of analysis requests -- export **OPAL_MONGODB_HOST**='datashield_mongo' # host of the mongo database used - -- export **POLL_QUEUE_SERVER**='' # e.g. '123.12.12.12:443' , queue server host and port(usually 443), the -q prefix is important. If you have a valid certificate and domain you will need to put the domain here instead of IP -- export **POLL_OPAL_SERVER**='' # e.g. '123.12.12.12:443' , opal server host and port, if left blank defaults to '-o datashield_opal:8443', which is correct for a pure docker setup -- export **POLL_THREADS**='' # e.g. '5' number of poll threads defaults to 2 -- export **CHECK_SERVER_CERT**='' # e.g. '-c' activate checking sever certificate - this adds extra security, however this results in your poll mechanism not working if your certificate is not correct or you have configured the *POLL_QUEUE_SERVER* using the IP - - -### Installing your own ssl certificate - - -**Certificate Opal Server** -To install your own ssl certificate for opal you can either use the user interface or the command line tools we have provided - -- command line tools: -1. exchange the opalcert.pem and opalkey.pem files with your own certificate files, but make sure, that the filename stays the same. -2. execute `./import_certificate.sh` in the opal_poll folder (NOT the one in the ./auth folder) -3. your certificate should now be updated - -- opal user interface -go to your opal server https://OPAL_SERVER_IP:443 and login as administrator using the password you set in the opal_poll.config file. - -**Intermediate Certificate Poll Server** - -*DO NOT CONTINUE HERE IF YOU CHOSE TO NOT ACTIVATE THE **CHECK_SERVER_CERT** OPTION* - -Install the needed certificate authority certificates - -In order for the poll module to check the certificate of the server, it needs to trust the ca_certificate used by your queue server. -To add a ca_cert (certificate authority ssl certificate) proceed as follows: -1. open the ./auth/ca_certs folder and replace the opalcacert.crt and the queuecacert.crt with your own ca cert files. -2. execute the ./addCaCertificates.sh - your certificates should now be installed on your local machine