From fd8a9a9b7bca8faa3ed10f46b7e9b8aec17716e2 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 30 Jan 2024 14:44:23 -0500 Subject: [PATCH 001/130] feat: fast and slow data directories for prod --- docs/installation.md | 7 +++++-- docs/troubleshooting.md | 2 +- etc/bento.env | 38 ++++++++++++++++++------------------ etc/bento_deploy.env | 6 ++++-- etc/bento_dev.env | 6 ++++-- etc/default_config.env | 6 ++++-- py_bentoctl/other_helpers.py | 3 ++- 7 files changed, 39 insertions(+), 29 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index b9e01b95..f1e62dec 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -51,8 +51,11 @@ BENTOV2_PRIVATE_MODE=false BENTO_CBIOPORTAL_ENABLED=false # --------------------------------------------------------------------- -# set this to a data storage location, optionally within the repo itself, like: /path-to-my-bentov2-repo/data -BENTOV2_ROOT_DATA_DIR=./data +# Set this to a data storage location, optionally within the repo itself, like: /path-to-my-bentov2-repo/data +# Data directories are split to better use SSD and HDD resources in prod. +# In dev/local it is more convenient to use a single directory +BENTO_FAST_DATA_DIR=./data +BENTO_SLOW_DATA_DIR=./data # Auth ---------------------------------------------------------------- # - Session secret should be set to a unique secure value. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 42dcb9de..6256de11 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -79,7 +79,7 @@ How to notice: Services are experiencing errors because they are unable to read This is often a permission being denied because the host paths of volumes are owned by `root`, verify with: ```shell -ls -la ${BENTOV2_ROOT_DATA_DIR} # For data volumes +ls -la data/ # For data volumes ls -la lib/ # For configuration volumes ``` diff --git a/etc/bento.env b/etc/bento.env index 3993e621..a4388470 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -48,7 +48,7 @@ BENTOV2_AUTH_CONTAINER_NAME=${BENTOV2_PREFIX}-auth BENTO_AUTH_NETWORK=${BENTOV2_PREFIX}-auth-net BENTOV2_AUTH_SERVICE_HOST=0.0.0.0 BENTOV2_AUTH_INTERNAL_PORT=8443 -BENTOV2_AUTH_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/auth/data +BENTOV2_AUTH_VOL_DIR=${BENTO_FAST_DATA_DIR}/auth/data BENTOV2_AUTH_WELLKNOWN_PATH=/realms/${BENTOV2_AUTH_REALM}/.well-known/openid-configuration BENTOV2_AUTH_REDIRECT_AFTER_LOGOUT_PATH=/realms/${BENTOV2_AUTH_REALM}/protocol/openid-connect/logout @@ -89,7 +89,7 @@ BENTO_AUTHZ_DB="authz" BENTO_AUTHZ_DB_USER="authz_user" # BENTO_AUTHZ_DB_PASSWORD is set in local.env BENTO_AUTHZ_DB_NETWORK=${BENTOV2_PREFIX}-authz-db-net -BENTO_AUTHZ_DB_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/authz/db +BENTO_AUTHZ_DB_VOL_DIR=${BENTO_FAST_DATA_DIR}/authz/db BENTO_AUTHZ_DB_MEM_LIM=1G # Web @@ -113,7 +113,7 @@ BENTOV2_DROP_BOX_CONTAINER_NAME=${BENTOV2_PREFIX}-drop-box BENTO_DROP_BOX_NETWORK=${BENTOV2_PREFIX}-drop-box-net BENTOV2_DROP_BOX_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTOV2_DROP_BOX_EXTERNAL_PORT=6000 -BENTOV2_DROP_BOX_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/drop-box/data-x +BENTOV2_DROP_BOX_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drop-box/data-x BENTO_DROP_BOX_FS_BASE_PATH='/data' # Internal base path for drop box mount + absolute paths BENTOV2_DROP_BOX_MEM_LIM=2G @@ -139,7 +139,7 @@ BENTOV2_NOTIFICATION_CONTAINER_NAME=${BENTOV2_PREFIX}-notification BENTO_NOTIFICATION_NETWORK=${BENTOV2_PREFIX}-notification-net BENTOV2_NOTIFICATION_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTOV2_NOTIFICATION_EXTERNAL_PORT=8500 -BENTOV2_NOTIFICATION_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/notification/data +BENTOV2_NOTIFICATION_VOL_DIR=${BENTO_FAST_DATA_DIR}/notification/data BENTO_NOTIFICATION_DEBUGGER_EXTERNAL_PORT=5681 BENTO_NOTIFICATION_DEBUGGER_INTERNAL_PORT=5681 @@ -190,7 +190,7 @@ BENTO_REFERENCE_DB_IMAGE=postgres BENTO_REFERENCE_DB_VERSION=16 BENTO_REFERENCE_DB_CONTAINER_NAME=${BENTOV2_PREFIX}-reference-db BENTO_REFERENCE_DB_NETWORK=${BENTOV2_PREFIX}-reference-db-net -BENTO_REFERENCE_DB_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/reference/data +BENTO_REFERENCE_DB_VOL_DIR=${BENTO_FAST_DATA_DIR}/reference/data BENTO_REFERENCE_DB_EXTERNAL_PORT=9512 BENTO_REFERENCE_DB_NAME="reference" BENTO_REFERENCE_DB_USER="reference_user" @@ -205,13 +205,13 @@ BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net BENTOV2_WES_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTOV2_WES_EXTERNAL_PORT=9250 -BENTOV2_WES_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/wes/data -BENTOV2_WES_VOL_TMP_DIR=${BENTOV2_ROOT_DATA_DIR}/wes/tmp +BENTOV2_WES_VOL_DIR=${BENTO_FAST_DATA_DIR}/wes/data +BENTOV2_WES_VOL_TMP_DIR=${BENTO_FAST_DATA_DIR}/wes/tmp BENTOV2_WES_PROD_DROP_BOX_VOL_DIR=${BENTOV2_DROP_BOX_VOL_DIR} # ( assumes a relative drop-box path like ./lib/drop-box/... - adjust accordingly ) -BENTOV2_WES_DEV_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/wes/dev/data -BENTOV2_WES_DEV_VOL_TMP_DIR=${BENTOV2_ROOT_DATA_DIR}/wes/dev/tmp +BENTOV2_WES_DEV_VOL_DIR=${BENTO_FAST_DATA_DIR}/wes/dev/data +BENTOV2_WES_DEV_VOL_TMP_DIR=${BENTO_FAST_DATA_DIR}/wes/dev/tmp BENTOV2_WES_DEV_DROP_BOX_VOL_DIR=${BENTOV2_DROP_BOX_VOL_DIR} BENTOV2_WES_DEBUGGER_INTERNAL_PORT=5680 @@ -236,14 +236,14 @@ BENTOV2_DRS_EXTERNAL_PORT=7007 BENTO_DRS_CONTAINER_DATA_VOLUME_DIR=/drs/bento_drs/data BENTO_DRS_CONTAINER_TMP_VOLUME_DIR=/drs/tmp -BENTOV2_DRS_PROD_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/drs/data +BENTOV2_DRS_PROD_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data BENTOV2_DRS_PROD_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} # ( assumes a relative wes path like ./lib/wes/... - adjust accordingly ) -BENTOV2_DRS_DEV_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/drs/data +BENTOV2_DRS_DEV_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data BENTOV2_DRS_DEV_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} -BENTO_DRS_TMP_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/drs/tmp +BENTO_DRS_TMP_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/tmp BENTOV2_DRS_DEBUGGER_INTERNAL_PORT=5682 BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT=5682 @@ -263,8 +263,8 @@ BENTOV2_KATSU_DB_USER=admin # BENTOV2_KATSU_DB_PASSWORD is set in local.env BENTOV2_KATSU_DB=metadata -BENTOV2_KATSU_DB_PROD_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/katsu/data -BENTOV2_KATSU_DB_DEV_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/katsu/dev/data +BENTOV2_KATSU_DB_PROD_VOL_DIR=${BENTO_FAST_DATA_DIR}/katsu/data +BENTOV2_KATSU_DB_DEV_VOL_DIR=${BENTO_FAST_DATA_DIR}/katsu/dev/data BENTOV2_KATSU_DB_MEM_LIM=4G BENTOV2_KATSU_DB_CPUS=4 @@ -299,7 +299,7 @@ BENTOV2_REDIS_CONTAINER_NAME=${BENTOV2_PREFIX}-redis BENTO_REDIS_NETWORK=${BENTOV2_PREFIX}-redis-net BENTOV2_REDIS_INTERNAL_PORT=6379 BENTOV2_REDIS_EXTERNAL_PORT=6379 -BENTOV2_REDIS_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/redis/data +BENTOV2_REDIS_VOL_DIR=${BENTO_FAST_DATA_DIR}/redis/data BENTOV2_REDIS_MEM_LIM=2G BENTOV2_REDIS_CPUS=2 @@ -310,7 +310,7 @@ BENTOV2_GOHAN_SERVICE_CONTACT=info@c3g.ca BENTOV2_GOHAN_SEMVER=5.0.0 # -- Host -BENTOV2_GOHAN_DATA_ROOT=${BENTOV2_ROOT_DATA_DIR}/gohan +BENTOV2_GOHAN_DATA_ROOT=${BENTO_FAST_DATA_DIR}/gohan # -- API BENTOV2_GOHAN_API_IMAGE=ghcr.io/bento-platform/gohan-api @@ -417,7 +417,7 @@ BENTO_CBIOPORTAL_IMAGE=cbioportal/cbioportal BENTO_CBIOPORTAL_IMAGE_VERSION=4.1.13 BENTO_CBIOPORTAL_INTERNAL_PORT=8080 BENTO_CBIOPORTAL_EXTERNAL_PORT=8089 -BENTO_CBIOPORTAL_STUDY_DIR=${BENTOV2_ROOT_DATA_DIR}/cbioportal/study +BENTO_CBIOPORTAL_STUDY_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/study BENTO_CBIOPORTAL_DATABASE_IMAGE=mysql BENTO_CBIOPORTAL_DATABASE_IMAGE_VERSION=5.7 @@ -427,7 +427,7 @@ BENTO_CBIOPORTAL_DATABASE_DBNAME=cbioportal BENTO_CBIOPORTAL_DATABASE_DBUSER=cbio_user # BENTO_CBIOPORTAL_DATABASE_PASSWORD=somepassword Should be set to secret values in local.env # BENTO_CBIOPORTAL_DATABASE_ROOT_PASSWORD=somepassword Should be set to secret values in local.env -BENTO_CBIOPORTAL_DATABASE_DATA_DIR=${BENTOV2_ROOT_DATA_DIR}/cbioportal/database +BENTO_CBIOPORTAL_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/database BENTO_CBIOPORTAL_SESSION_IMAGE=cbioportal/session-service BENTO_CBIOPORTAL_SESSION_IMAGE_VERSION=0.5.0 @@ -440,5 +440,5 @@ BENTO_CBIOPORTAL_SESSION_SERVER_PORT=5000 BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE=mongo BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.4 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db -BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTOV2_ROOT_DATA_DIR}/cbioportal/session_db +BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 7c95d668..ff8c14d5 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -14,9 +14,11 @@ BENTO_GOHAN_ENABLED='true' # - non-default values allow multiple deployments on the same machine. BENTOV2_PREFIX=bentov2 -# Root data directory on host. Bind volumes for services will be put +# Previously BENTOV2_ROOT_DATA_DIR, now split for SSD/HDD optimization in prod +# Root data directories on host. Bind volumes for services will be put # here as subdirectories and mounted into the relevant containers. -BENTOV2_ROOT_DATA_DIR=~/bentov2/data +BENTO_FAST_DATA_DIR=/app # /app is mounted to an SSD disk +BENTO_SLOW_DATA_DIR=/data # /data is mounted to an HDD disk # Gateway/domains ----------------------------------------------------- BENTOV2_DOMAIN=bentov2.local diff --git a/etc/bento_dev.env b/etc/bento_dev.env index cb8bf9be..af9c5b6f 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -10,9 +10,11 @@ BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' # Feature switches end ------------------------------------------------ -# Root data directory on host. Bind volumes for services will be put +# Previously BENTOV2_ROOT_DATA_DIR, now split for SSD/HDD optimization in prod +# Root data directories on host. Bind volumes for services will be put # here as subdirectories and mounted into the relevant containers. -BENTOV2_ROOT_DATA_DIR=${PWD}/data +BENTO_FAST_DATA_DIR=${PWD}/data +BENTO_SLOW_DATA_DIR=${PWD}/data # Gateway/domains ----------------------------------------------------- BENTOV2_DOMAIN=bentov2.local diff --git a/etc/default_config.env b/etc/default_config.env index da9b4ef5..74549cbd 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -23,9 +23,11 @@ BENTO_GIT_CLONE_HTTPS=0 # - non-default values allow multiple deployments on the same machine. BENTOV2_PREFIX=bentov2 -# Root data directory on host. Bind volumes for services will be put +# Previously BENTOV2_ROOT_DATA_DIR, now split for SSD/HDD optimization in prod +# Root data directories on host. Bind volumes for services will be put # here as subdirectories and mounted into the relevant containers. -BENTOV2_ROOT_DATA_DIR=~/bentov2/data +BENTO_FAST_DATA_DIR=${PWD}/data +BENTO_SLOW_DATA_DIR=${PWD}/data # Gateway/domains ----------------------------------------------------- BENTOV2_DOMAIN=bentov2.local diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 6cdf9983..80c54e02 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -190,7 +190,8 @@ def init_self_signed_certs(force: bool): def init_dirs(): data_dir_vars = { - "root": "BENTOV2_ROOT_DATA_DIR", + "root_fast": "BENTO_FAST_DATA_DIR", + "root_slow": "BENTO_SLOW_DATA_DIR", "authz-db": "BENTO_AUTHZ_DB_VOL_DIR", "drs-data": "BENTOV2_DRS_DEV_VOL_DIR" if c.DEV_MODE else "BENTOV2_DRS_PROD_VOL_DIR", "drs-tmp": "BENTO_DRS_TMP_VOL_DIR", From 357e4343f5e68c2caedfb199a7dd638065ae5ccf Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 30 Jan 2024 15:13:36 -0500 Subject: [PATCH 002/130] docs(migration): data directories migration guide --- README.md | 1 + docs/migrating_to_16.md | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 docs/migrating_to_16.md diff --git a/README.md b/README.md index 43ee24f7..b6261b82 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ that make up the Bento platform. ### Migration documents +* [v15 to v16](./docs/migrating_to_16.md) * [v14 to v15](./docs/migrating_to_15.md) * [Converting Phenopackets from V1 to V2 using `bentoctl`](./docs/phenopackets_v1_to_v2.md) * [v13 to v14](./docs/migrating_to_14.md) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md new file mode 100644 index 00000000..283c9377 --- /dev/null +++ b/docs/migrating_to_16.md @@ -0,0 +1,63 @@ +# Migrating to Bento v16 + +Bento v16 may now use 2 data directories for better resource usage in production. + + +The environment variable `BENTOV2_ROOT_DATA_DIR` was replaced with: +* `BENTO_FAST_DATA_DIR` for SSD mounts +* `BENTO_SLOW_DATA_DIR` for HDD mounts + + +By default, `Drop-Box` and `DRS` are now configured to use `BENTO_SLOW_DATA_DIR` as their data directory. + +All other services use `BENTO_FAST_DATA_DIR`. + +If needed, the default service specific volume directories defined in [bento.env](../etc/bento.env) can be overriden +in your `local.env`. + +## 1. Pre-update data config + +Before updating, perform the following steps: + +* Shut down Bento with `./bentoctl.bash stop` +* In `local.env`, replace `BENTOV2_ROOT_DATA_DIR` with `BENTO_FAST_DATA_DIR` and `BENTO_SLOW_DATA_DIR` + +To minimize side effects in local environments, we recommend that you use the same directory as before for both new variables. + +```bash +# local.env + +# Old +BENTOV2_ROOT_DATA_DIR=some_dir + +# Recommended new variables +BENTO_FAST_DATA_DIR=some_dir +BENTO_SLOW_DATA_DIR=some_dir +``` + +*IMPORTANT:* If you decide to split the data in distinct directories, make sure to migrate the data accordingly. + + +## 2. Update images + +```bash +./bentoctl.bash pull +``` + + +## 3. Create new Docker volume directories + +This step is optional if you DIDN'T split the data directories in step 1. + +```bash +./bentoctl.bash init-dirs # new volume directory for DRS temporary files + +# migrate the old data +``` + + +## 4. Restart Bento + +```bash +./bentoctl.bash start +``` From 108ab7eca6abd070bf7969f4cf5ad93131e7f602 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Wed, 31 Jan 2024 09:58:44 -0500 Subject: [PATCH 003/130] chore: drs tmp dir in fast data --- docs/troubleshooting.md | 3 ++- etc/bento.env | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6256de11..3d6cefa7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -79,7 +79,8 @@ How to notice: Services are experiencing errors because they are unable to read This is often a permission being denied because the host paths of volumes are owned by `root`, verify with: ```shell -ls -la data/ # For data volumes +ls -la ${BENTO_SLOW_DATA_DIR} # For "slow" data volumes +ls -la ${BENTO_FAST_DATA_DIR} # For "fast" data volumes ls -la lib/ # For configuration volumes ``` diff --git a/etc/bento.env b/etc/bento.env index a4388470..5771dee8 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -243,7 +243,7 @@ BENTOV2_DRS_PROD_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} BENTOV2_DRS_DEV_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data BENTOV2_DRS_DEV_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} -BENTO_DRS_TMP_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/tmp +BENTO_DRS_TMP_VOL_DIR=${BENTO_FAST_DATA_DIR}/drs/tmp BENTOV2_DRS_DEBUGGER_INTERNAL_PORT=5682 BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT=5682 From faa3230b69bf209268f60f08a23bec89834fc638 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 27 Feb 2024 15:15:29 -0500 Subject: [PATCH 004/130] chore: new compose structure to address changes in compose 2.24 --- README.md | 4 +- docker-compose.dev.yaml | 60 +---------------------------- docker-compose.local.yaml | 55 +-------------------------- docker-compose.prod.yaml | 3 -- docker-compose.yaml | 79 +++++++-------------------------------- 5 files changed, 18 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 1fd510b9..0e5e061d 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ that make up the Bento platform. ## Requirements -- Docker >= 19.03.8 -- Docker Compose >= 2.14.0, <2.24 (plugin form: you should have the `docker compose` command available, without a dash) +- Docker >= 24.0.4 +- Docker Compose >= 2.20.0 (plugin form: you should have the `docker compose` command available, without a dash) - Python >= 3.9 (for `bentoctl`); the services require Python 3.10 but this is included in their Docker images. diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 2e2837c9..8cb0d634 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,18 +1,12 @@ -# extends main docker-compose.yml and various service-specific compose files; +# merges with main docker-compose.yml and various service-specific compose files; # adds in development/debug environment variables, debugpy ports services: web: - extends: - file: docker-compose.yaml - service: web environment: - NODE_ENV=development public: - extends: - file: $PWD/lib/public/docker-compose.public.yaml - service: public environment: - NODE_ENV=development ports: @@ -91,23 +85,12 @@ services: - ${BENTOV2_DOMAIN} - ${BENTOV2_PORTAL_DOMAIN} - ${BENTOV2_AUTH_DOMAIN} - extends: - file: docker-compose.yaml - service: gateway authz: - extends: - file: docker-compose.yaml - service: authz environment: - BENTO_DEBUG=True katsu: - depends_on: - - katsu-db - extends: - file: $PWD/lib/katsu/docker-compose.katsu.yaml - service: katsu volumes: - ${BENTOV2_KATSU_DEV_WES_VOL_TMP_DIR}:/wes/tmp:ro environment: @@ -119,9 +102,6 @@ services: - "${BENTOV2_KATSU_DEBUGGER_EXTERNAL_PORT}:${BENTOV2_KATSU_DEBUGGER_INTERNAL_PORT}" katsu-db: - extends: - file: $PWD/lib/katsu/docker-compose.katsu.yaml - service: katsu-db ports: - "${BENTOV2_KATSU_DB_EXTERNAL_PORT}:${BENTOV2_KATSU_DB_INTERNAL_PORT}" volumes: @@ -155,9 +135,6 @@ services: - LOG_LEVEL=debug - PORT=${BENTOV2_AGGREGATION_INTERNAL_PORT} - DEBUGGER_PORT=${BENTOV2_AGGREGATION_DEBUGGER_INTERNAL_PORT} - extends: - file: $PWD/lib/aggregation/docker-compose.aggregation.yaml - service: aggregation ports: - "${BENTOV2_AGGREGATION_EXTERNAL_PORT}:${BENTOV2_AGGREGATION_INTERNAL_PORT}" - "${BENTOV2_AGGREGATION_DEBUGGER_EXTERNAL_PORT}:${BENTOV2_AGGREGATION_DEBUGGER_INTERNAL_PORT}" @@ -169,9 +146,6 @@ services: - FLASK_DEBUG=True - BENTO_VALIDATE_SSL=False - DEBUGGER_PORT=${BENTOV2_WES_DEBUGGER_EXTERNAL_PORT} - extends: - file: $PWD/lib/wes/docker-compose.wes.yaml - service: wes volumes: - ${BENTOV2_WES_DEV_VOL_DIR}:/wes/data - ${BENTOV2_WES_DEV_VOL_TMP_DIR}:/wes/tmp @@ -184,9 +158,6 @@ services: environment: - FLASK_DEBUG=True - DEBUGGER_PORT=${BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT} - extends: - file: $PWD/lib/drs/docker-compose.drs.yaml - service: drs ports: - "${BENTOV2_DRS_EXTERNAL_PORT}:5000" - "${BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT}:${BENTOV2_DRS_DEBUGGER_INTERNAL_PORT}" @@ -202,9 +173,6 @@ services: - BENTO_DEBUG=True - CHORD_DEBUG=True - BENTO_BEACON_DEBUG=true - extends: - file: $PWD/lib/beacon/docker-compose.beacon.yaml - service: beacon ports: - "${BENTO_BEACON_EXTERNAL_PORT}:${BENTO_BEACON_INTERNAL_PORT}" - "${BENTO_BEACON_DEBUGGER_EXTERNAL_PORT}:${BENTO_BEACON_DEBUGGER_INTERNAL_PORT}" @@ -218,9 +186,6 @@ services: - LOG_LEVEL=debug ports: - "${BENTOV2_SERVICE_REGISTRY_EXTERNAL_PORT}:${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT}" - extends: - file: docker-compose.yaml - service: service-registry drop-box: environment: @@ -228,9 +193,6 @@ services: - BENTO_DEBUG=True ports: - "${BENTOV2_DROP_BOX_EXTERNAL_PORT}:${BENTOV2_DROP_BOX_INTERNAL_PORT}" - extends: - file: docker-compose.yaml - service: drop-box notification: environment: @@ -240,17 +202,11 @@ services: ports: - "${BENTOV2_NOTIFICATION_EXTERNAL_PORT}:${BENTOV2_NOTIFICATION_INTERNAL_PORT}" - "${BENTO_NOTIFICATION_DEBUGGER_EXTERNAL_PORT}:${BENTO_NOTIFICATION_DEBUGGER_INTERNAL_PORT}" - extends: - file: docker-compose.yaml - service: notification event-relay: environment: - BENTO_DEBUG=true - NODE_ENV=development - extends: - file: docker-compose.yaml - service: event-relay reference: environment: @@ -259,14 +215,8 @@ services: - LOG_LEVEL=debug ports: - "${BENTO_REFERENCE_EXTERNAL_PORT}:${BENTO_REFERENCE_INTERNAL_PORT}" - extends: - file: $PWD/lib/reference/docker-compose.reference.yaml - service: reference reference-db: - extends: - file: $PWD/lib/reference/docker-compose.reference.yaml - service: reference-db ports: - "${BENTO_REFERENCE_DB_EXTERNAL_PORT}:5432" @@ -278,11 +228,3 @@ services: "http.cors.enabled": "true" "http.cors.allow-origin": "http://localhost:8081" "http.cors.allow-headers": X-Requested-With,Content-Type,Content-Length,Authorization - extends: - file: $PWD/lib/gohan/docker-compose.gohan.yaml - service: gohan-elasticsearch - - gohan-api: - extends: - file: $PWD/lib/gohan/docker-compose.gohan.yaml - service: gohan-api diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index b8ea8ffc..038caf3f 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -1,4 +1,4 @@ -# extends docker-compose.dev.yaml; adds in repository volumes + git configuration +# merges with docker-compose.yaml and docker-compose.dev.yaml; adds in repository volumes + git configuration services: web: @@ -10,9 +10,6 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/web - extends: - file: docker-compose.dev.yaml - service: web public: image: ${BENTO_PUBLIC_IMAGE}:${BENTO_PUBLIC_VERSION_DEV} @@ -23,9 +20,6 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/bento-public - extends: - file: docker-compose.dev.yaml - service: public profiles: - public @@ -38,14 +32,8 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/gateway - extends: - file: docker-compose.dev.yaml - service: gateway authz: - extends: - file: docker-compose.dev.yaml - service: authz image: ${BENTO_AUTHZ_IMAGE}:${BENTO_AUTHZ_VERSION_DEV} volumes: - ./repos/authz:/authorization @@ -55,9 +43,6 @@ services: - BENTO_GIT_REPOSITORY_DIR=/authorization katsu: - extends: - file: docker-compose.dev.yaml - service: katsu image: ${BENTOV2_KATSU_IMAGE}:${BENTOV2_KATSU_VERSION_DEV} depends_on: - katsu-db @@ -69,18 +54,10 @@ services: - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/app - katsu-db: - extends: - file: docker-compose.dev.yaml - service: katsu-db - adminer: # No Docker networks required, bound to host ports: - 8080:8080 - extends: - file: docker-compose.dev.yaml - service: adminer aggregation: image: ${BENTOV2_AGGREGATION_IMAGE}:${BENTOV2_AGGREGATION_VERSION_DEV} @@ -88,9 +65,6 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/aggregation - extends: - file: docker-compose.dev.yaml - service: aggregation volumes: - ./repos/aggregation:/aggregation @@ -101,9 +75,6 @@ services: - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/wes - CELERY_DEBUG=true - extends: - file: docker-compose.dev.yaml - service: wes volumes: - ./repos/wes:/wes @@ -113,9 +84,6 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/drs - extends: - file: docker-compose.dev.yaml - service: drs volumes: - ./repos/drs:/drs @@ -126,9 +94,6 @@ services: - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/beacon - DEBUGGER_PORT=${BENTO_BEACON_DEBUGGER_INTERNAL_PORT} - extends: - file: docker-compose.dev.yaml - service: beacon volumes: - ./repos/beacon:/beacon profiles: @@ -144,9 +109,6 @@ services: - BENTO_GIT_NAME - BENTO_GIT_EMAIL - BENTO_GIT_REPOSITORY_DIR=/service_registry - extends: - file: docker-compose.dev.yaml - service: service-registry drop-box: image: ${BENTOV2_DROP_BOX_IMAGE}:${BENTOV2_DROP_BOX_VERSION_DEV} @@ -156,9 +118,6 @@ services: - BENTO_GIT_REPOSITORY_DIR=/drop-box volumes: - ./repos/drop-box:/drop-box - extends: - file: docker-compose.dev.yaml - service: drop-box notification: image: ${BENTOV2_NOTIFICATION_IMAGE}:${BENTOV2_NOTIFICATION_VERSION_DEV} @@ -168,9 +127,6 @@ services: - BENTO_GIT_REPOSITORY_DIR=/notification volumes: - ./repos/notification:/notification - extends: - file: docker-compose.dev.yaml - service: notification event-relay: image: ${BENTOV2_EVENT_RELAY_IMAGE}:${BENTOV2_EVENT_RELAY_VERSION_DEV} @@ -180,9 +136,6 @@ services: - BENTO_GIT_REPOSITORY_DIR=/app volumes: - ./repos/event-relay:/app - extends: - file: docker-compose.dev.yaml - service: event-relay reference: image: ${BENTO_REFERENCE_IMAGE}:${BENTO_REFERENCE_VERSION_DEV} @@ -192,9 +145,6 @@ services: - BENTO_GIT_REPOSITORY_DIR=/reference volumes: - ./repos/reference:/reference - extends: - file: docker-compose.dev.yaml - service: reference gohan-api: image: ${BENTOV2_GOHAN_API_IMAGE}:${BENTOV2_GOHAN_API_VERSION_DEV} @@ -206,6 +156,3 @@ services: volumes: - ./repos/gohan-api:/gohan-api - ./repos/gohan-api/src/api/workflows:/app/workflows - extends: - file: docker-compose.dev.yaml - service: gohan-api diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 294cb7e8..547a3d0f 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,8 +1,5 @@ services: web: - extends: - file: docker-compose.yaml - service: web environment: - NODE_ENV=production volumes: diff --git a/docker-compose.yaml b/docker-compose.yaml index 9ad353ee..169feae9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,3 +1,12 @@ +include: + - path: $PWD/lib/aggregation/docker-compose.aggregation.yaml + - path: $PWD/lib/authz/docker-compose.authz.yaml + - path: $PWD/lib/drs/docker-compose.drs.yaml + - path: $PWD/lib/katsu/docker-compose.katsu.yaml + - path: $PWD/lib/reference/docker-compose.reference.yaml + - path: $PWD/lib/web/docker-compose.web.yaml + - path: $PWD/lib/wes/docker-compose.wes.yaml + services: gateway: image: ${BENTOV2_GATEWAY_IMAGE}:${BENTOV2_GATEWAY_VERSION} @@ -160,39 +169,10 @@ services: # AUTH: - # auth and auth-db included via extra compose file: lib/auth/docker-compose.auth.yaml + # auth and auth-db included via extra, optional compose file: lib/auth/docker-compose.auth.yaml # PUBLIC: - # public included via extra compose file: lib/public/docker-compose.public.yaml - - - authz: - extends: - file: $PWD/lib/authz/docker-compose.authz.yaml - service: authz - - authz-db: - extends: - file: $PWD/lib/authz/docker-compose.authz.yaml - service: authz-db - - - web: - extends: - file: $PWD/lib/web/docker-compose.web.yaml - service: web - - - katsu: - extends: - file: $PWD/lib/katsu/docker-compose.katsu.yaml - service: katsu - - katsu-db: - extends: - file: $PWD/lib/katsu/docker-compose.katsu.yaml - service: katsu-db - + # public included via extra, optional compose file: lib/public/docker-compose.public.yaml drop-box: image: ${BENTOV2_DROP_BOX_IMAGE}:${BENTOV2_DROP_BOX_VERSION} @@ -266,14 +246,6 @@ services: interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - drs: - networks: - - drs-net - extends: - file: $PWD/lib/drs/docker-compose.drs.yaml - service: drs - - notification: image: ${BENTOV2_NOTIFICATION_IMAGE}:${BENTOV2_NOTIFICATION_VERSION} container_name: ${BENTOV2_NOTIFICATION_CONTAINER_NAME} @@ -303,12 +275,6 @@ services: interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - aggregation: - extends: - file: $PWD/lib/aggregation/docker-compose.aggregation.yaml - service: aggregation - - event-relay: image: ${BENTOV2_EVENT_RELAY_IMAGE}:${BENTOV2_EVENT_RELAY_VERSION} container_name: ${BENTOV2_EVENT_RELAY_CONTAINER_NAME} @@ -365,23 +331,6 @@ services: interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - reference: - extends: - file: $PWD/lib/reference/docker-compose.reference.yaml - service: reference - - reference-db: - extends: - file: $PWD/lib/reference/docker-compose.reference.yaml - service: reference-db - - - wes: - extends: - file: $PWD/lib/wes/docker-compose.wes.yaml - service: wes - - redis: image: ${BENTOV2_REDIS_BASE_IMAGE}:${BENTOV2_REDIS_BASE_IMAGE_VERSION} container_name: ${BENTOV2_REDIS_CONTAINER_NAME} @@ -400,13 +349,13 @@ services: interval: 15s # Faster interval for services which others may require to be healthy # BEACON: - # beacon included via extra compose file: lib/beacon/docker-compose.beacon.yaml + # beacon included via extra, optional compose file: lib/beacon/docker-compose.beacon.yaml # GOHAN - # gohan-api and gohan-elasticsearch included via extra compose file: lib/gohan/docker-compose.gohan.yaml + # gohan-api and gohan-elasticsearch included via extra, optional compose file: lib/gohan/docker-compose.gohan.yaml # cBioPortal - # `cbioportal` included via extra compose file: lib/cbioportal/docker-compose.cbioportal.yaml + # `cbioportal` included via extra, optional compose file: lib/cbioportal/docker-compose.cbioportal.yaml # Lots of networks! Here, we have more or less one network per service, plus a few ones for databases From aed83be2e0206a1b5695c52ed7be5c5c421c7861 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Wed, 28 Feb 2024 15:44:27 -0500 Subject: [PATCH 005/130] compose auth env vars on public container --- lib/public/docker-compose.public.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/public/docker-compose.public.yaml b/lib/public/docker-compose.public.yaml index 798a9136..cab16ddf 100644 --- a/lib/public/docker-compose.public.yaml +++ b/lib/public/docker-compose.public.yaml @@ -19,6 +19,9 @@ services: - BENTO_BEACON_UI_ENABLED - BEACON_URL=${BENTOV2_PUBLIC_URL}/api/beacon - INTERNAL_PORT=${BENTO_PUBLIC_INTERNAL_PORT} + - BENTO_PUBLIC_URL=${BENTOV2_PUBLIC_URL} + - OPENID_CONFIG_URL=${BENTOV2_AUTH_PUBLIC_URL}${BENTOV2_AUTH_WELLKNOWN_PATH} + - CLIENT_ID=${BENTOV2_AUTH_CLIENT_ID} expose: - "${BENTO_PUBLIC_INTERNAL_PORT}" volumes: From 60c79a2d38ee27fb1e7f89a69380b6d737af6558 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 28 Feb 2024 16:47:21 -0500 Subject: [PATCH 006/130] chore: update cbioportal and related configuration --- etc/bento.env | 8 ++++---- lib/cbioportal/config/portal.properties | 5 +---- lib/cbioportal/docker-compose.cbioportal.yaml | 2 -- py_bentoctl/services.py | 1 + 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 0b94ca16..2c0cc92b 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=0.10.2 +BENTOV2_GATEWAY_VERSION=pr-11 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway @@ -416,7 +416,7 @@ BENTO_BEACON_OIDC_ISSUER=${BENTOV2_AUTH_PUBLIC_URL}/auth/realms/${BENTOV2_AUTH_R BENTO_CBIOPORTAL_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal BENTO_CBIOPORTAL_NETWORK=${BENTOV2_PREFIX}-cbioportal-net BENTO_CBIOPORTAL_IMAGE=cbioportal/cbioportal -BENTO_CBIOPORTAL_IMAGE_VERSION=4.1.13 +BENTO_CBIOPORTAL_IMAGE_VERSION=5.4.10 BENTO_CBIOPORTAL_INTERNAL_PORT=8080 BENTO_CBIOPORTAL_EXTERNAL_PORT=8089 BENTO_CBIOPORTAL_STUDY_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/study @@ -432,7 +432,7 @@ BENTO_CBIOPORTAL_DATABASE_DBUSER=cbio_user BENTO_CBIOPORTAL_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/database BENTO_CBIOPORTAL_SESSION_IMAGE=cbioportal/session-service -BENTO_CBIOPORTAL_SESSION_IMAGE_VERSION=0.5.0 +BENTO_CBIOPORTAL_SESSION_IMAGE_VERSION=0.6.2 BENTO_CBIOPORTAL_SESSION_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session BENTO_CBIOPORTAL_SESSION_NETWORK=${BENTOV2_PREFIX}-cbioportal-session-net BENTO_CBIOPORTAL_SESSION_SERVER_PORT=5000 @@ -440,7 +440,7 @@ BENTO_CBIOPORTAL_SESSION_SERVER_PORT=5000 # Need mongo 6.0+ to support arm64 + x86_64 (both needed for development) # Technically not in line with the cBioPortal requirements... we'll see if it causes issues BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE=mongo -BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.4 +BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.7 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network diff --git a/lib/cbioportal/config/portal.properties b/lib/cbioportal/config/portal.properties index 949a7b0e..fd03421f 100644 --- a/lib/cbioportal/config/portal.properties +++ b/lib/cbioportal/config/portal.properties @@ -6,10 +6,8 @@ app.name=cbioportal # database db.user=cbio_user db.password=somepassword -db.host=cbioportal-db:3306 -db.portal_db_name=cbioportal db.driver=com.mysql.jdbc.Driver -db.connection_string=jdbc:mysql://cbioportal-db:3306/ +db.connection_string=jdbc:mysql://cbioportal-db:3306/cbioportal?useSSL=false&allowPublicKeyRetrieval=true # set tomcat_resource_name when using dbconnector=jndi instead of the default # dbconnector=dbcp. Note that dbconnector needs to be set in CATLINA_OPTS when # using Tomcat (CATALINA_OPTS="-Ddbconnector=jndi"). It does not get picked up @@ -20,7 +18,6 @@ db.connection_string=jdbc:mysql://cbioportal-db:3306/ # this should normally be set to false. In some cases you could set this to true (e.g. for testing a feature of a newer release that is not related to the schema change in expected db version above): db.suppress_schema_version_mismatch_errors=false -db.use_ssl=false # web page cosmetics skin.title=cBioPortal for Cancer Genomics skin.email_contact=cbioportal at googlegroups dot com diff --git a/lib/cbioportal/docker-compose.cbioportal.yaml b/lib/cbioportal/docker-compose.cbioportal.yaml index d1f5bcfc..01d7d29c 100644 --- a/lib/cbioportal/docker-compose.cbioportal.yaml +++ b/lib/cbioportal/docker-compose.cbioportal.yaml @@ -1,7 +1,6 @@ services: cbioportal: image: ${BENTO_CBIOPORTAL_IMAGE}:${BENTO_CBIOPORTAL_IMAGE_VERSION} - platform: linux/amd64 # No arm64 build container_name: ${BENTO_CBIOPORTAL_CONTAINER_NAME} environment: SHOW_DEBUG_INFO: "true" @@ -51,7 +50,6 @@ services: cbioportal-session: image: ${BENTO_CBIOPORTAL_SESSION_IMAGE}:${BENTO_CBIOPORTAL_SESSION_IMAGE_VERSION} - platform: linux/amd64 # No arm64 build container_name: ${BENTO_CBIOPORTAL_SESSION_CONTAINER_NAME} environment: SERVER_PORT: ${BENTO_CBIOPORTAL_SESSION_SERVER_PORT} diff --git a/py_bentoctl/services.py b/py_bentoctl/services.py index 68683d11..1d259535 100644 --- a/py_bentoctl/services.py +++ b/py_bentoctl/services.py @@ -78,6 +78,7 @@ def _get_service_specific_compose(service: str): "authz": ("BENTO_AUTHZ_IMAGE", "BENTO_AUTHZ_VERSION", "BENTO_AUTHZ_VERSION_DEV"), "authz-db": ("BENTO_AUTHZ_DB_IMAGE", "BENTO_AUTHZ_DB_VERSION", None), "beacon": ("BENTO_BEACON_IMAGE", "BENTO_BEACON_VERSION", "BENTO_BEACON_VERSION_DEV"), + "cbioportal": ("BENTO_CBIOPORTAL_IMAGE", "BENTO_CBIOPORTAL_IMAGE_VERSION", None), "drop-box": ("BENTOV2_DROP_BOX_IMAGE", "BENTOV2_DROP_BOX_VERSION", "BENTOV2_DROP_BOX_VERSION_DEV"), "drs": ("BENTOV2_DRS_IMAGE", "BENTOV2_DRS_VERSION", "BENTOV2_DRS_VERSION_DEV"), "event-relay": ("BENTOV2_EVENT_RELAY_IMAGE", "BENTOV2_EVENT_RELAY_VERSION", "BENTOV2_EVENT_RELAY_VERSION_DEV"), From f19de79086ac026ea3ae2a6011db7c83c3342fcf Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 4 Mar 2024 15:22:30 -0500 Subject: [PATCH 007/130] set bento public to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 0b94ca16..d05897e4 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -375,7 +375,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=0.17.3 +BENTO_PUBLIC_VERSION=edge BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 0050b22c01d52da51125abbe50ff581d802792f1 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 11 Mar 2024 14:07:08 -0400 Subject: [PATCH 008/130] bump redis to 7.0.15 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index d05897e4..b255791b 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -296,7 +296,7 @@ CHORD_METADATA_SUB_PATH=/api/metadata # Redis BENTOV2_REDIS_BASE_IMAGE=redis -BENTOV2_REDIS_BASE_IMAGE_VERSION=7.0.14-alpine +BENTOV2_REDIS_BASE_IMAGE_VERSION=7.0.15-alpine BENTOV2_REDIS_CONTAINER_NAME=${BENTOV2_PREFIX}-redis BENTO_REDIS_NETWORK=${BENTOV2_PREFIX}-redis-net BENTOV2_REDIS_INTERNAL_PORT=6379 From 0cde2a2b77ed228998793480805a3e5cecab83f2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 11 Mar 2024 15:08:46 -0400 Subject: [PATCH 009/130] set web to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index b255791b..d7342dde 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=4.1.4 +BENTOV2_WEB_VERSION=edge BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net From 62e451641730d661618e2e2f263d960e31913e75 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 12:25:42 -0400 Subject: [PATCH 010/130] chore: remove redundant $PWD in compose file includes --- docker-compose.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 23f1bd82..dd8ad616 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,11 +1,11 @@ include: - - path: $PWD/lib/aggregation/docker-compose.aggregation.yaml - - path: $PWD/lib/authz/docker-compose.authz.yaml - - path: $PWD/lib/drs/docker-compose.drs.yaml - - path: $PWD/lib/katsu/docker-compose.katsu.yaml - - path: $PWD/lib/reference/docker-compose.reference.yaml - - path: $PWD/lib/web/docker-compose.web.yaml - - path: $PWD/lib/wes/docker-compose.wes.yaml + - path: lib/aggregation/docker-compose.aggregation.yaml + - path: lib/authz/docker-compose.authz.yaml + - path: lib/drs/docker-compose.drs.yaml + - path: lib/katsu/docker-compose.katsu.yaml + - path: lib/reference/docker-compose.reference.yaml + - path: lib/web/docker-compose.web.yaml + - path: lib/wes/docker-compose.wes.yaml services: gateway: From a5ea8e3b36fa7265267b230a8b8f7616c946014c Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 12:41:48 -0400 Subject: [PATCH 011/130] feat(bentoctl): add `compose config` command wrapper --- py_bentoctl/entry.py | 16 +++++++++++++--- py_bentoctl/services.py | 5 +++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/py_bentoctl/entry.py b/py_bentoctl/entry.py index 51114e9d..a2519352 100644 --- a/py_bentoctl/entry.py +++ b/py_bentoctl/entry.py @@ -14,7 +14,7 @@ from typing import Optional, Tuple, Type -__version__ = "0.5.1" +__version__ = "0.6.0" class SubCommand(ABC): @@ -198,6 +198,13 @@ def exec(args): s.logs_service(args.service, args.follow) +class ComposeConfig(SubCommand): + + @staticmethod + def exec(args): + s.compose_config() + + # Other helpers class InitDirs(SubCommand): @@ -323,6 +330,10 @@ def _add_subparser(arg: str, help_text: str, subcommand: Type[SubCommand], alias # Feature-specific initialization commands _add_subparser("init-cbioportal", "Initialize cBioPortal if enabled", InitCBioPortal) + # Other commands + _add_subparser("convert-pheno", + "Convert a Phenopacket V1 JSON document to V2", ConvertPhenopacket, aliases=("conv",)) + # Service commands _add_subparser("run", "Run Bento services.", Run, aliases=("start", "up")) _add_subparser("stop", "Stop Bento services.", Stop, aliases=("down",)) @@ -339,8 +350,7 @@ def _add_subparser(arg: str, help_text: str, subcommand: Type[SubCommand], alias _add_subparser("shell", "Run a shell inside an already-running service container.", Shell, aliases=("sh",)) _add_subparser("run-as-shell", "Run a shell inside a stopped service container.", RunShell) _add_subparser("logs", "Check logs for a service.", Logs) - _add_subparser("convert-pheno", - "Convert a Phenopacket V1 JSON document to V2", ConvertPhenopacket, aliases=("conv",)) + _add_subparser("compose-config", "Generate Compose config YAML.", ComposeConfig) p_args = parser.parse_args(args) diff --git a/py_bentoctl/services.py b/py_bentoctl/services.py index 68683d11..290f46dd 100644 --- a/py_bentoctl/services.py +++ b/py_bentoctl/services.py @@ -19,6 +19,7 @@ "enter_shell_for_service", "run_as_shell_for_service", "logs_service", + "compose_config", ] BENTO_SERVICES_DATA_BY_KIND = { @@ -404,3 +405,7 @@ def logs_service(compose_service: str, follow: bool) -> None: os.execvp( c.COMPOSE[0], (*_get_service_specific_compose(compose_service), "logs", *extra_args, compose_service)) + + +def compose_config() -> None: + os.execvp(c.COMPOSE[0], (*c.COMPOSE, "config")) From a7b4c9b87fe9de1a1aa6be0966972e3294b2f44d Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 13:37:33 -0400 Subject: [PATCH 012/130] feat(bentoctl): add --services flag to compose-config command --- py_bentoctl/entry.py | 6 +++++- py_bentoctl/services.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/py_bentoctl/entry.py b/py_bentoctl/entry.py index a2519352..3038c056 100644 --- a/py_bentoctl/entry.py +++ b/py_bentoctl/entry.py @@ -200,9 +200,13 @@ def exec(args): class ComposeConfig(SubCommand): + @staticmethod + def add_args(sp): + sp.add_argument("--services", action="store_true", help="List services seen by Compose.") + @staticmethod def exec(args): - s.compose_config() + s.compose_config(args.services) # Other helpers diff --git a/py_bentoctl/services.py b/py_bentoctl/services.py index 290f46dd..36158040 100644 --- a/py_bentoctl/services.py +++ b/py_bentoctl/services.py @@ -407,5 +407,5 @@ def logs_service(compose_service: str, follow: bool) -> None: (*_get_service_specific_compose(compose_service), "logs", *extra_args, compose_service)) -def compose_config() -> None: - os.execvp(c.COMPOSE[0], (*c.COMPOSE, "config")) +def compose_config(services_flag: bool) -> None: + os.execvp(c.COMPOSE[0], (*c.COMPOSE, "config", *(("--services",) if services_flag else ()))) From ea6b584da22a5a4bf232e9a7f446662212561f07 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:28:55 -0400 Subject: [PATCH 013/130] factor out drop box compose config to new file --- docker-compose.yaml | 1 + lib/drop-box/docker-compose.drop-box.yaml | 30 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 lib/drop-box/docker-compose.drop-box.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index dd8ad616..3f32b2bc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,7 @@ include: - path: lib/aggregation/docker-compose.aggregation.yaml - path: lib/authz/docker-compose.authz.yaml + - path: lib/drop-box/docker-compose.drop-box.yaml - path: lib/drs/docker-compose.drs.yaml - path: lib/katsu/docker-compose.katsu.yaml - path: lib/reference/docker-compose.reference.yaml diff --git a/lib/drop-box/docker-compose.drop-box.yaml b/lib/drop-box/docker-compose.drop-box.yaml new file mode 100644 index 00000000..aee98855 --- /dev/null +++ b/lib/drop-box/docker-compose.drop-box.yaml @@ -0,0 +1,30 @@ +services: + drop-box: + image: ${BENTOV2_DROP_BOX_IMAGE}:${BENTOV2_DROP_BOX_VERSION} + container_name: ${BENTOV2_DROP_BOX_CONTAINER_NAME} + environment: + - BENTO_UID + - BENTO_DEBUG=False + - SERVICE_DATA=${BENTO_DROP_BOX_FS_BASE_PATH} + - SERVICE_URL=${BENTOV2_PUBLIC_URL}/api/drop-box + - CORS_ORIGINS=${BENTO_CORS_ORIGINS} + - BENTO_AUTHZ_SERVICE_URL + - OPENID_CONFIG_URL=${BENTO_OPENID_CONFIG_URL} + networks: + - drop-box-net + expose: + - ${BENTOV2_DROP_BOX_INTERNAL_PORT} + volumes: + - ${BENTOV2_DROP_BOX_VOL_DIR}:${BENTO_DROP_BOX_FS_BASE_PATH} + mem_limit: ${BENTOV2_DROP_BOX_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_DROP_BOX_CPUS} + cpu_shares: 1024 + healthcheck: + test: [ "CMD", "curl", "http://localhost:${BENTOV2_DROP_BOX_INTERNAL_PORT}/service-info" ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: ${BENTOV2_HEALTHCHECK_INTERVAL} + +networks: + drop-box-net: + external: true + name: ${BENTO_DROP_BOX_NETWORK} From 880044618e0b1233ae247722792e36d0c1b7fab6 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:29:06 -0400 Subject: [PATCH 014/130] fix: bad network decl in public compose config --- lib/public/docker-compose.public.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/public/docker-compose.public.yaml b/lib/public/docker-compose.public.yaml index 798a9136..598f7ece 100644 --- a/lib/public/docker-compose.public.yaml +++ b/lib/public/docker-compose.public.yaml @@ -37,9 +37,9 @@ services: - public networks: - katsu-db-net: + katsu-net: external: true - name: ${BENTO_KATSU_DB_NETWORK} + name: ${BENTO_KATSU_NETWORK} public-net: external: true name: ${BENTO_PUBLIC_NETWORK} From ef94045aaaa99b07b8b837cfd7277feaed54801f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:29:33 -0400 Subject: [PATCH 015/130] rm drop box from main compose --- docker-compose.yaml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3f32b2bc..9f493c9d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -176,31 +176,6 @@ services: # PUBLIC: # public included via extra, optional compose file: lib/public/docker-compose.public.yaml - drop-box: - image: ${BENTOV2_DROP_BOX_IMAGE}:${BENTOV2_DROP_BOX_VERSION} - container_name: ${BENTOV2_DROP_BOX_CONTAINER_NAME} - environment: - - BENTO_UID - - BENTO_DEBUG=False - - SERVICE_DATA=${BENTO_DROP_BOX_FS_BASE_PATH} - - SERVICE_URL=${BENTOV2_PUBLIC_URL}/api/drop-box - - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - - BENTO_AUTHZ_SERVICE_URL - - OPENID_CONFIG_URL=${BENTO_OPENID_CONFIG_URL} - networks: - - drop-box-net - expose: - - ${BENTOV2_DROP_BOX_INTERNAL_PORT} - volumes: - - ${BENTOV2_DROP_BOX_VOL_DIR}:${BENTO_DROP_BOX_FS_BASE_PATH} - mem_limit: ${BENTOV2_DROP_BOX_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_DROP_BOX_CPUS} - cpu_shares: 1024 - healthcheck: - test: [ "CMD", "curl", "http://localhost:${BENTOV2_DROP_BOX_INTERNAL_PORT}/service-info" ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - service-registry: image: ${BENTOV2_SERVICE_REGISTRY_IMAGE}:${BENTOV2_SERVICE_REGISTRY_VERSION} From 414b4e8e752c858be65d6b8a1fa77be33e4ab6cd Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:34:52 -0400 Subject: [PATCH 016/130] move service-registry to separate compose file --- docker-compose.yaml | 65 ++++--------------- .../docker-compose.service-registry.yaml | 50 ++++++++++++++ 2 files changed, 61 insertions(+), 54 deletions(-) create mode 100644 lib/service-registry/docker-compose.service-registry.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index 9f493c9d..2d2c3afc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,12 +1,15 @@ include: - - path: lib/aggregation/docker-compose.aggregation.yaml - - path: lib/authz/docker-compose.authz.yaml - - path: lib/drop-box/docker-compose.drop-box.yaml - - path: lib/drs/docker-compose.drs.yaml - - path: lib/katsu/docker-compose.katsu.yaml - - path: lib/reference/docker-compose.reference.yaml - - path: lib/web/docker-compose.web.yaml - - path: lib/wes/docker-compose.wes.yaml + - path: + - lib/aggregation/docker-compose.aggregation.yaml + - lib/authz/docker-compose.authz.yaml + - lib/drop-box/docker-compose.drop-box.yaml + - lib/drs/docker-compose.drs.yaml + - lib/katsu/docker-compose.katsu.yaml + - lib/reference/docker-compose.reference.yaml + - lib/service-registry/docker-compose.service-registry.yaml + - lib/web/docker-compose.web.yaml + - lib/wes/docker-compose.wes.yaml + project_directory: ../.. services: gateway: @@ -177,52 +180,6 @@ services: # public included via extra, optional compose file: lib/public/docker-compose.public.yaml - service-registry: - image: ${BENTOV2_SERVICE_REGISTRY_IMAGE}:${BENTOV2_SERVICE_REGISTRY_VERSION} - container_name: ${BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME} - environment: - - BENTO_UID - - # Following the revised bento_services.json schema - # (see Python typing in bento_service_registry) - # A list of services on a single domain which are registered in the service - # registry instance. - - BENTO_SERVICES=/bento_services.json - - # Common URL base for all services - - BENTO_URL=${BENTOV2_PORTAL_PUBLIC_URL} - - # Variables for generating service URLs from URL templates found in bento_services.json - - BENTO_PUBLIC_URL=${BENTOV2_PUBLIC_URL} - - BENTO_PORTAL_PUBLIC_URL=${BENTOV2_PORTAL_PUBLIC_URL} - - - BENTO_DEBUG=False - - BENTO_VALIDATE_SSL=True - - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - - BENTO_AUTHZ_SERVICE_URL - - # Timeout, in seconds (integers only), for contacting services from the JSON - - CONTACT_TIMEOUT=10 - - # Service ID for the /service-info endpoint - - SERVICE_ID=ca.c3g.bento:${BENTOV2_SERVICE_REGISTRY_VERSION} - - - INTERNAL_PORT=${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} - networks: - - service-registry-net - expose: - - ${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} - volumes: - - $PWD/etc/bento_services.json:/bento_services.json:ro - mem_limit: ${BENTOV2_SERVICE_REGISTRY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_SERVICE_REGISTRY_CPUS} - cpu_shares: 256 - healthcheck: - test: [ "CMD", "curl", "http://localhost:${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT}/service-info" ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - - notification: image: ${BENTOV2_NOTIFICATION_IMAGE}:${BENTOV2_NOTIFICATION_VERSION} container_name: ${BENTOV2_NOTIFICATION_CONTAINER_NAME} diff --git a/lib/service-registry/docker-compose.service-registry.yaml b/lib/service-registry/docker-compose.service-registry.yaml new file mode 100644 index 00000000..a41cdae6 --- /dev/null +++ b/lib/service-registry/docker-compose.service-registry.yaml @@ -0,0 +1,50 @@ +services: + service-registry: + image: ${BENTOV2_SERVICE_REGISTRY_IMAGE}:${BENTOV2_SERVICE_REGISTRY_VERSION} + container_name: ${BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME} + environment: + - BENTO_UID + + # Following the revised bento_services.json schema + # (see Python typing in bento_service_registry) + # A list of services on a single domain which are registered in the service + # registry instance. + - BENTO_SERVICES=/bento_services.json + + # Common URL base for all services + - BENTO_URL=${BENTOV2_PORTAL_PUBLIC_URL} + + # Variables for generating service URLs from URL templates found in bento_services.json + - BENTO_PUBLIC_URL=${BENTOV2_PUBLIC_URL} + - BENTO_PORTAL_PUBLIC_URL=${BENTOV2_PORTAL_PUBLIC_URL} + + - BENTO_DEBUG=False + - BENTO_VALIDATE_SSL=True + - CORS_ORIGINS=${BENTO_CORS_ORIGINS} + - BENTO_AUTHZ_SERVICE_URL + + # Timeout, in seconds (integers only), for contacting services from the JSON + - CONTACT_TIMEOUT=10 + + # Service ID for the /service-info endpoint + - SERVICE_ID=ca.c3g.bento:${BENTOV2_SERVICE_REGISTRY_VERSION} + + - INTERNAL_PORT=${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} + networks: + - service-registry-net + expose: + - ${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} + volumes: + - etc/bento_services.json:/bento_services.json:ro + mem_limit: ${BENTOV2_SERVICE_REGISTRY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_SERVICE_REGISTRY_CPUS} + cpu_shares: 256 + healthcheck: + test: [ "CMD", "curl", "http://localhost:${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT}/service-info" ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: ${BENTOV2_HEALTHCHECK_INTERVAL} + +networks: + service-registry-net: + external: true + name: ${BENTO_SERVICE_REGISTRY_NETWORK} From b7fc091a5e3b9342a95ff20b24e1665c009433d1 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:35:03 -0400 Subject: [PATCH 017/130] add more optional feature compose files to includes --- docker-compose.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 2d2c3afc..d2b1d8f7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,15 +1,20 @@ include: - path: - lib/aggregation/docker-compose.aggregation.yaml + - lib/auth/docker-compose.auth.yaml # Optional feature; controlled by a compose profile - lib/authz/docker-compose.authz.yaml + - lib/beacon/docker-compose.beacon.yaml # Optional feature; controlled by a compose profile + - lib/cbioportal/docker-compose.cbioportal.yaml # Optional feature; controlled by a compose profile - lib/drop-box/docker-compose.drop-box.yaml - lib/drs/docker-compose.drs.yaml + - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml + - lib/public/docker-compose.public.yaml # Optional feature; controlled by a compose profile - lib/reference/docker-compose.reference.yaml - lib/service-registry/docker-compose.service-registry.yaml - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml - project_directory: ../.. + project_directory: ../.. # Paths in the lib/* compose files must be relative to the Bento base directory services: gateway: From 1712c2300cce16095e1ab9002f0301d8cd5784e8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:38:18 -0400 Subject: [PATCH 018/130] move notification to separate compose file --- docker-compose.yaml | 45 ------------------- .../docker-compose.notification.yaml | 36 +++++++++++++++ 2 files changed, 36 insertions(+), 45 deletions(-) create mode 100644 lib/notification/docker-compose.notification.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index d2b1d8f7..c3a38101 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -178,42 +178,6 @@ services: interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - # AUTH: - # auth and auth-db included via extra, optional compose file: lib/auth/docker-compose.auth.yaml - - # PUBLIC: - # public included via extra, optional compose file: lib/public/docker-compose.public.yaml - - - notification: - image: ${BENTOV2_NOTIFICATION_IMAGE}:${BENTOV2_NOTIFICATION_VERSION} - container_name: ${BENTOV2_NOTIFICATION_CONTAINER_NAME} - environment: - - BENTO_UID - - FLASK_DEBUG=False - - FLASK_APP=app - - DATABASE=/notification/data - - REDIS_HOST=${BENTOV2_REDIS_CONTAINER_NAME} - - REDIS_PORT=${BENTOV2_REDIS_INTERNAL_PORT} - - INTERNAL_PORT=${BENTOV2_NOTIFICATION_INTERNAL_PORT} - - BENTO_AUTHZ_SERVICE_URL - - CORS_ORIGINS=${BENTOV2_PUBLIC_URL};${BENTOV2_PORTAL_PUBLIC_URL} - networks: - - notification-net - - redis-net # For event pub/sub - expose: - - ${BENTOV2_NOTIFICATION_INTERNAL_PORT} - volumes: - - ${BENTOV2_NOTIFICATION_VOL_DIR}:/notification/data - mem_limit: ${BENTOV2_NOTIFICATION_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_NOTIFICATION_CPUS} - cpu_shares: 512 - healthcheck: - test: [ "CMD", "curl", "http://localhost:${BENTOV2_NOTIFICATION_INTERNAL_PORT}/service-info" ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - - event-relay: image: ${BENTOV2_EVENT_RELAY_IMAGE}:${BENTOV2_EVENT_RELAY_VERSION} container_name: ${BENTOV2_EVENT_RELAY_CONTAINER_NAME} @@ -287,15 +251,6 @@ services: timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} interval: 15s # Faster interval for services which others may require to be healthy - # BEACON: - # beacon included via extra, optional compose file: lib/beacon/docker-compose.beacon.yaml - - # GOHAN - # gohan-api and gohan-elasticsearch included via extra, optional compose file: lib/gohan/docker-compose.gohan.yaml - - # cBioPortal - # `cbioportal` included via extra, optional compose file: lib/cbioportal/docker-compose.cbioportal.yaml - # Lots of networks! Here, we have more or less one network per service, plus a few ones for databases # (which don't even get exposed to the gateway) and some other extras. diff --git a/lib/notification/docker-compose.notification.yaml b/lib/notification/docker-compose.notification.yaml new file mode 100644 index 00000000..9198f7a5 --- /dev/null +++ b/lib/notification/docker-compose.notification.yaml @@ -0,0 +1,36 @@ +services: + notification: + image: ${BENTOV2_NOTIFICATION_IMAGE}:${BENTOV2_NOTIFICATION_VERSION} + container_name: ${BENTOV2_NOTIFICATION_CONTAINER_NAME} + environment: + - BENTO_UID + - FLASK_DEBUG=False + - FLASK_APP=app + - DATABASE=/notification/data + - REDIS_HOST=${BENTOV2_REDIS_CONTAINER_NAME} + - REDIS_PORT=${BENTOV2_REDIS_INTERNAL_PORT} + - INTERNAL_PORT=${BENTOV2_NOTIFICATION_INTERNAL_PORT} + - BENTO_AUTHZ_SERVICE_URL + - CORS_ORIGINS=${BENTOV2_PUBLIC_URL};${BENTOV2_PORTAL_PUBLIC_URL} + networks: + - notification-net + - redis-net # For event pub/sub + expose: + - ${BENTOV2_NOTIFICATION_INTERNAL_PORT} + volumes: + - ${BENTOV2_NOTIFICATION_VOL_DIR}:/notification/data + mem_limit: ${BENTOV2_NOTIFICATION_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_NOTIFICATION_CPUS} + cpu_shares: 512 + healthcheck: + test: [ "CMD", "curl", "http://localhost:${BENTOV2_NOTIFICATION_INTERNAL_PORT}/service-info" ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: ${BENTOV2_HEALTHCHECK_INTERVAL} + +networks: + notification-net: + external: true + name: ${BENTO_NOTIFICATION_NETWORK} + redis-net: + external: true + name: ${BENTO_REDIS_NETWORK} From 4137e90a1fd96348ecf39be33b8d2a96d3008e60 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 14:47:42 -0400 Subject: [PATCH 019/130] move rest of services into compose files + fix enabled services --- docker-compose.yaml | 94 +------------------ .../docker-compose.event-relay.yaml | 66 +++++++++++++ lib/redis/docker-compose.redis.yaml | 22 +++++ .../docker-compose.service-registry.yaml | 2 +- py_bentoctl/config.py | 27 ++++-- 5 files changed, 114 insertions(+), 97 deletions(-) create mode 100644 lib/event-relay/docker-compose.event-relay.yaml create mode 100644 lib/redis/docker-compose.redis.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index c3a38101..c8d29691 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,14 +7,17 @@ include: - lib/cbioportal/docker-compose.cbioportal.yaml # Optional feature; controlled by a compose profile - lib/drop-box/docker-compose.drop-box.yaml - lib/drs/docker-compose.drs.yaml + - lib/event-relay/docker-compose.event-relay.yaml - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml + - lib/notification/docker-compose.notification.yaml - lib/public/docker-compose.public.yaml # Optional feature; controlled by a compose profile + - lib/redis/docker-compose.redis.yaml - lib/reference/docker-compose.reference.yaml - lib/service-registry/docker-compose.service-registry.yaml - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml - project_directory: ../.. # Paths in the lib/* compose files must be relative to the Bento base directory + project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory services: gateway: @@ -178,80 +181,6 @@ services: interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - event-relay: - image: ${BENTOV2_EVENT_RELAY_IMAGE}:${BENTOV2_EVENT_RELAY_VERSION} - container_name: ${BENTOV2_EVENT_RELAY_CONTAINER_NAME} - environment: - # Default to production BENTO_DEBUG and NODE_ENV - - BENTO_DEBUG=false - - NODE_ENV=production - - # If true, JSON_MESSAGES will parse all messages recieved from the subscription - # and de-serialize them before passing them to the socket.io connection. - # e.g. If the message {"test": true} is passed with JSON_MESSAGES on, the - # corresponding socket.io message will be: - # {"message": {"test": true}, "channel": "chord.something"} - # Otherwise, the message will be: - # {"message": "{\"test\": true}", "channel": "chord.something"} - - JSON_MESSAGES=true - - # socket.io "path" for the server created by the service. - - SOCKET_IO_PATH=${BENTOV2_EVENT_RELAY_SOCKET_IO_PATH} - - # CORS origins for connecting to the socket - - CORS_ORIGINS=${BENTOV2_PUBLIC_URL};${BENTOV2_PORTAL_PUBLIC_URL} - - # Connection string (URL or path to UNIX socket file) for the Redis instance. - - REDIS_CONNECTION=redis://${BENTOV2_REDIS_CONTAINER_NAME}:${BENTOV2_REDIS_INTERNAL_PORT} - - # Subscription pattern the Redis PubSub connection. The default is configured - # to be chord.* for the Bento platform, but it can be set to anything. - # See https://redis.io/topics/pubsub and specifically the pattern-matching. - - REDIS_SUBSCRIBE_PATTERN=bento.* - - # Where the service will listen for requests. Can be a UNIX socket path or a - # port number (e.g. 8080, which is the default value if nothing is set.) - # Will also check the SERVICE_SOCKET environment variable for legacy reasons - # related to the Bento platform. - # Use the standard Bento service internal port, overriding the default - - SERVICE_LISTEN_ON=${BENTO_STD_SERVICE_INTERNAL_PORT} - - # Pass in authz service URL for authorizing socket connections - - BENTO_AUTHZ_SERVICE_URL=${BENTO_AUTHZ_SERVICE_URL} - depends_on: - - redis - networks: - - event-relay-net - - redis-net # For event pub/sub - expose: - - ${BENTOV2_EVENT_RELAY_INTERNAL_PORT} - mem_limit: ${BENTOV2_EVENT_RELAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_EVENT_RELAY_CPUS} - cpu_shares: 512 - healthcheck: - test: [ "CMD", "curl", "http://localhost:${BENTOV2_EVENT_RELAY_INTERNAL_PORT}/service-info" ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - - - redis: - image: ${BENTOV2_REDIS_BASE_IMAGE}:${BENTOV2_REDIS_BASE_IMAGE_VERSION} - container_name: ${BENTOV2_REDIS_CONTAINER_NAME} - networks: - - redis-net - expose: - - ${BENTOV2_REDIS_INTERNAL_PORT} - volumes: - - ${BENTOV2_REDIS_VOL_DIR}:/data - mem_limit: ${BENTOV2_REDIS_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_REDIS_CPUS} - cpu_shares: 512 - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: 15s # Faster interval for services which others may require to be healthy - - # Lots of networks! Here, we have more or less one network per service, plus a few ones for databases # (which don't even get exposed to the gateway) and some other extras. # @@ -279,9 +208,6 @@ networks: authz-net: external: true name: ${BENTO_AUTHZ_NETWORK} - authz-db-net: - external: true - name: ${BENTO_AUTHZ_DB_NETWORK} beacon-net: external: true name: ${BENTO_BEACON_NETWORK} @@ -300,30 +226,18 @@ networks: gohan-api-net: external: true name: ${BENTO_GOHAN_API_NETWORK} - gohan-es-net: - external: true - name: ${BENTO_GOHAN_ES_NETWORK} katsu-net: external: true name: ${BENTO_KATSU_NETWORK} - katsu-db-net: - external: true - name: ${BENTO_KATSU_DB_NETWORK} notification-net: external: true name: ${BENTO_NOTIFICATION_NETWORK} public-net: external: true name: ${BENTO_PUBLIC_NETWORK} - redis-net: - external: true - name: ${BENTO_REDIS_NETWORK} reference-net: external: true name: ${BENTO_REFERENCE_NETWORK} - reference-db-net: - external: true - name: ${BENTO_REFERENCE_DB_NETWORK} service-registry-net: external: true name: ${BENTO_SERVICE_REGISTRY_NETWORK} diff --git a/lib/event-relay/docker-compose.event-relay.yaml b/lib/event-relay/docker-compose.event-relay.yaml new file mode 100644 index 00000000..f78f59c9 --- /dev/null +++ b/lib/event-relay/docker-compose.event-relay.yaml @@ -0,0 +1,66 @@ +include: + - path: lib/redis/docker-compose.redis.yaml + +services: + event-relay: + image: ${BENTOV2_EVENT_RELAY_IMAGE}:${BENTOV2_EVENT_RELAY_VERSION} + container_name: ${BENTOV2_EVENT_RELAY_CONTAINER_NAME} + environment: + # Default to production BENTO_DEBUG and NODE_ENV + - BENTO_DEBUG=false + - NODE_ENV=production + + # If true, JSON_MESSAGES will parse all messages recieved from the subscription + # and de-serialize them before passing them to the socket.io connection. + # e.g. If the message {"test": true} is passed with JSON_MESSAGES on, the + # corresponding socket.io message will be: + # {"message": {"test": true}, "channel": "chord.something"} + # Otherwise, the message will be: + # {"message": "{\"test\": true}", "channel": "chord.something"} + - JSON_MESSAGES=true + + # socket.io "path" for the server created by the service. + - SOCKET_IO_PATH=${BENTOV2_EVENT_RELAY_SOCKET_IO_PATH} + + # CORS origins for connecting to the socket + - CORS_ORIGINS=${BENTOV2_PUBLIC_URL};${BENTOV2_PORTAL_PUBLIC_URL} + + # Connection string (URL or path to UNIX socket file) for the Redis instance. + - REDIS_CONNECTION=redis://${BENTOV2_REDIS_CONTAINER_NAME}:${BENTOV2_REDIS_INTERNAL_PORT} + + # Subscription pattern the Redis PubSub connection. The default is configured + # to be chord.* for the Bento platform, but it can be set to anything. + # See https://redis.io/topics/pubsub and specifically the pattern-matching. + - REDIS_SUBSCRIBE_PATTERN=bento.* + + # Where the service will listen for requests. Can be a UNIX socket path or a + # port number (e.g. 8080, which is the default value if nothing is set.) + # Will also check the SERVICE_SOCKET environment variable for legacy reasons + # related to the Bento platform. + # Use the standard Bento service internal port, overriding the default + - SERVICE_LISTEN_ON=${BENTO_STD_SERVICE_INTERNAL_PORT} + + # Pass in authz service URL for authorizing socket connections + - BENTO_AUTHZ_SERVICE_URL=${BENTO_AUTHZ_SERVICE_URL} + depends_on: + - redis + networks: + - event-relay-net + - redis-net # For event pub/sub + expose: + - ${BENTOV2_EVENT_RELAY_INTERNAL_PORT} + mem_limit: ${BENTOV2_EVENT_RELAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_EVENT_RELAY_CPUS} + cpu_shares: 512 + healthcheck: + test: [ "CMD", "curl", "http://localhost:${BENTOV2_EVENT_RELAY_INTERNAL_PORT}/service-info" ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: ${BENTOV2_HEALTHCHECK_INTERVAL} + +networks: + event-relay-net: + external: true + name: ${BENTO_EVENT_RELAY_NETWORK} + redis-net: + external: true + name: ${BENTO_REDIS_NETWORK} diff --git a/lib/redis/docker-compose.redis.yaml b/lib/redis/docker-compose.redis.yaml new file mode 100644 index 00000000..de903b92 --- /dev/null +++ b/lib/redis/docker-compose.redis.yaml @@ -0,0 +1,22 @@ +services: + redis: + image: ${BENTOV2_REDIS_BASE_IMAGE}:${BENTOV2_REDIS_BASE_IMAGE_VERSION} + container_name: ${BENTOV2_REDIS_CONTAINER_NAME} + networks: + - redis-net + expose: + - ${BENTOV2_REDIS_INTERNAL_PORT} + volumes: + - ${BENTOV2_REDIS_VOL_DIR}:/data + mem_limit: ${BENTOV2_REDIS_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_REDIS_CPUS} + cpu_shares: 512 + healthcheck: + test: [ "CMD", "redis-cli", "ping" ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: 15s # Faster interval for services which others may require to be healthy + +networks: + redis-net: + external: true + name: ${BENTO_REDIS_NETWORK} diff --git a/lib/service-registry/docker-compose.service-registry.yaml b/lib/service-registry/docker-compose.service-registry.yaml index a41cdae6..a966b3ab 100644 --- a/lib/service-registry/docker-compose.service-registry.yaml +++ b/lib/service-registry/docker-compose.service-registry.yaml @@ -35,7 +35,7 @@ services: expose: - ${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} volumes: - - etc/bento_services.json:/bento_services.json:ro + - ./etc/bento_services.json:/bento_services.json:ro mem_limit: ${BENTOV2_SERVICE_REGISTRY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_SERVICE_REGISTRY_CPUS} cpu_shares: 256 diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index 5bf20490..9a8e64ff 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -5,10 +5,13 @@ import os import pathlib import pwd +import subprocess import yaml from typing import Dict, Generator, List, Optional, Tuple +from .utils import err + __all__ = [ "DOCKER_COMPOSE_FILE_BASE", "DOCKER_COMPOSE_FILE_DEV", @@ -127,13 +130,25 @@ def __init__(self, enabled: bool, profile: str): COMPOSE: Tuple[str, ...] = ("docker", "compose") -def _get_enabled_services(compose_file: str, filter_out: Tuple[str, ...] = ()) -> Generator[str, None, None]: +def _get_enabled_services( + compose_files: Tuple[str, ...], + filter_out: Tuple[str, ...] = (), +) -> Generator[str, None, None]: # Loop through compose file and find enabled services - either no profiles specified, # or the profile of an enabled feature. - # Load base docker-compose services list - with open(compose_file) as dcf: - compose_data = yaml.load(dcf, yaml.Loader) + # Generate merged Docker Compose YAML using docker compose config command + r = subprocess.run( + (*COMPOSE, *itertools.chain.from_iterable(("-f", cf) for cf in compose_files), "config"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if r.returncode != 0: + err(f"Parsing compose file(s) {compose_files} failed with error:\n\t{r.stderr.decode('utf-8').strip()}") + exit(1) + + # Load the compose data + compose_data = yaml.load(r.stdout, yaml.Loader) for k, v in compose_data["services"].items(): if k in filter_out: @@ -150,7 +165,7 @@ def _get_enabled_services(compose_file: str, filter_out: Tuple[str, ...] = ()) - break -BASE_SERVICES: Tuple[str, ...] = tuple(itertools.chain.from_iterable(_get_enabled_services(cf, ()) for cf in ( +BASE_SERVICES: Tuple[str, ...] = tuple(itertools.chain.from_iterable(_get_enabled_services((cf,), ()) for cf in ( DOCKER_COMPOSE_FILE_BASE, DOCKER_COMPOSE_FILE_AUTH, DOCKER_COMPOSE_FILE_BEACON, @@ -162,7 +177,7 @@ def _get_enabled_services(compose_file: str, filter_out: Tuple[str, ...] = ()) - # Load dev docker-compose services list if in DEV_MODE DOCKER_COMPOSE_DEV_SERVICES: Tuple[str, ...] = tuple( - _get_enabled_services(DOCKER_COMPOSE_FILE_DEV, BASE_SERVICES)) if DEV_MODE else () + _get_enabled_services((DOCKER_COMPOSE_FILE_BASE, DOCKER_COMPOSE_FILE_DEV), BASE_SERVICES)) if DEV_MODE else () # Final amalgamation of services for Bento taking into account dev/prod mode + profiles DOCKER_COMPOSE_SERVICES: Tuple[str, ...] = BASE_SERVICES + DOCKER_COMPOSE_DEV_SERVICES From 43c7bd822772f03a93b3f4473249d6dd57495186 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 15:35:07 -0400 Subject: [PATCH 020/130] fix(bentoctl): properly handle profiles when parsing compose yaml --- py_bentoctl/config.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index 9a8e64ff..2a697ee7 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -117,7 +117,7 @@ def __init__(self, enabled: bool, profile: str): BENTO_FEATURE_GOHAN = BentoOptionalFeature( enabled=_env_get_bool("BENTO_GOHAN_ENABLED", default=False), profile="gohan") BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public") -BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=BENTO_DOMAIN_REDIRECT, profile="redirect") +BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect") if not DEV_MODE and BENTO_FEATURE_CBIOPORTAL.enabled: import sys @@ -137,9 +137,15 @@ def _get_enabled_services( # Loop through compose file and find enabled services - either no profiles specified, # or the profile of an enabled feature. + enabled_profiles = tuple(get_enabled_feature_profiles()) + # Generate merged Docker Compose YAML using docker compose config command r = subprocess.run( - (*COMPOSE, *itertools.chain.from_iterable(("-f", cf) for cf in compose_files), "config"), + ( + *COMPOSE, + *itertools.chain.from_iterable(("-f", cf) for cf in compose_files), + *itertools.chain.from_iterable(("--profile", p) for p in enabled_profiles), + "config"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -160,9 +166,9 @@ def _get_enabled_services( yield k continue for p in profiles: - if (f := BENTO_OPTIONAL_FEATURES_BY_PROFILE.get(p)) is not None and f.enabled: + if p in enabled_profiles: yield k - break + break # escape profile loop, we found one which enables this service BASE_SERVICES: Tuple[str, ...] = tuple(itertools.chain.from_iterable(_get_enabled_services((cf,), ()) for cf in ( From 8e7113a2de57fc74d0cf31fe2df0738d69c8292c Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 12 Mar 2024 16:18:38 -0400 Subject: [PATCH 021/130] remove redundant merging of optional service compose files --- py_bentoctl/config.py | 23 +---------------------- py_bentoctl/services.py | 12 ++++-------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index 2a697ee7..e4b589f8 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -18,12 +18,6 @@ "DOCKER_COMPOSE_FILE_LOCAL", "DOCKER_COMPOSE_FILE_PROD", - "DOCKER_COMPOSE_FILE_AUTH", - "DOCKER_COMPOSE_FILE_BEACON", - "DOCKER_COMPOSE_FILE_CBIOPORTAL", - "DOCKER_COMPOSE_FILE_GOHAN", - "DOCKER_COMPOSE_FILE_PUBLIC", - "DOCKER_COMPOSE_SERVICES", "DOCKER_COMPOSE_DEV_SERVICES", @@ -63,13 +57,6 @@ DOCKER_COMPOSE_FILE_LOCAL = "./docker-compose.local.yaml" DOCKER_COMPOSE_FILE_PROD = "./docker-compose.prod.yaml" -DOCKER_COMPOSE_FILE_AUTH = "./lib/auth/docker-compose.auth.yaml" -DOCKER_COMPOSE_FILE_BEACON = "./lib/beacon/docker-compose.beacon.yaml" -DOCKER_COMPOSE_FILE_CBIOPORTAL = "./lib/cbioportal/docker-compose.cbioportal.yaml" -DOCKER_COMPOSE_FILE_GOHAN = "./lib/gohan/docker-compose.gohan.yaml" -DOCKER_COMPOSE_FILE_PUBLIC = "./lib/public/docker-compose.public.yaml" -DOCKER_COMPOSE_FILE_REFERENCE = "./lib/reference/docker-compose.reference.yaml" - USER = os.getenv("USER") BENTO_UID: int = int(os.getenv("BENTO_UID", str(os.getuid()))) @@ -171,15 +158,7 @@ def _get_enabled_services( break # escape profile loop, we found one which enables this service -BASE_SERVICES: Tuple[str, ...] = tuple(itertools.chain.from_iterable(_get_enabled_services((cf,), ()) for cf in ( - DOCKER_COMPOSE_FILE_BASE, - DOCKER_COMPOSE_FILE_AUTH, - DOCKER_COMPOSE_FILE_BEACON, - DOCKER_COMPOSE_FILE_CBIOPORTAL, - DOCKER_COMPOSE_FILE_GOHAN, - DOCKER_COMPOSE_FILE_PUBLIC, - DOCKER_COMPOSE_FILE_REFERENCE, -))) +BASE_SERVICES: Tuple[str, ...] = tuple(_get_enabled_services((DOCKER_COMPOSE_FILE_BASE,), ())) # Load dev docker-compose services list if in DEV_MODE DOCKER_COMPOSE_DEV_SERVICES: Tuple[str, ...] = tuple( diff --git a/py_bentoctl/services.py b/py_bentoctl/services.py index 36158040..7ba474a4 100644 --- a/py_bentoctl/services.py +++ b/py_bentoctl/services.py @@ -3,7 +3,7 @@ import subprocess from termcolor import cprint -from typing import Callable, Dict, Optional, Tuple +from typing import Callable, Dict, Literal, Optional, Tuple from . import config as c from .state import MODE_LOCAL, MODE_PREBUILT, get_state, set_state_services @@ -52,12 +52,8 @@ def _get_compose_with_files(dev: bool = False, local: bool = False): *c.COMPOSE, "-f", c.DOCKER_COMPOSE_FILE_BASE, - # Service/feature-specific compose files - profiles will take care of actual service enable/disable - "-f", c.DOCKER_COMPOSE_FILE_AUTH, - "-f", c.DOCKER_COMPOSE_FILE_BEACON, - "-f", c.DOCKER_COMPOSE_FILE_CBIOPORTAL, - "-f", c.DOCKER_COMPOSE_FILE_GOHAN, - "-f", c.DOCKER_COMPOSE_FILE_PUBLIC, + # Service/feature-specific compose files are not listed here + # - includes + profiles will take care of incorporation + actual service enable/disable *(("-f", c.DOCKER_COMPOSE_FILE_DEV) if dev else ("-f", c.DOCKER_COMPOSE_FILE_PROD)), *(("-f", c.DOCKER_COMPOSE_FILE_LOCAL) if local else ()), @@ -295,7 +291,7 @@ def mode_service(compose_service: str) -> None: exit(1) mode = service_state[compose_service]["mode"] - colour = "green" if mode == MODE_PREBUILT else "blue" + colour: Literal["green", "blue"] = "green" if mode == MODE_PREBUILT else "blue" if mode == MODE_PREBUILT and not c.DEV_MODE: mode += "\t(prod)" else: From ec1e0e639df836f8cd480f17ead6c27b35dd565f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 13 Mar 2024 10:42:58 -0400 Subject: [PATCH 022/130] fix: bad bind path for katsu config.json --- lib/katsu/docker-compose.katsu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/katsu/docker-compose.katsu.yaml b/lib/katsu/docker-compose.katsu.yaml index 33a9f6fc..331c8816 100644 --- a/lib/katsu/docker-compose.katsu.yaml +++ b/lib/katsu/docker-compose.katsu.yaml @@ -42,7 +42,7 @@ services: # target: metadata_db_secret volumes: # Custom Katsu config - - ./config.json:${BENTOV2_KATSU_INTERNAL_PROJECT_DIRECTORY}/chord_metadata_service/config.json:ro + - ./lib/katsu/config.json:${BENTOV2_KATSU_INTERNAL_PROJECT_DIRECTORY}/chord_metadata_service/config.json:ro # Temp WES dir (read only mount, since we are just pulling files from it) - ${BENTOV2_KATSU_PROD_WES_VOL_TMP_DIR}:/wes/tmp:ro mem_limit: ${BENTOV2_KATSU_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 From b5791a7768a00d5488e3044ebdef7e9e35d8c134 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 13 Mar 2024 10:51:24 -0400 Subject: [PATCH 023/130] refact: move gateway compose def to separate file --- docker-compose.yaml | 230 +----------------------- lib/gateway/docker-compose.gateway.yaml | 228 +++++++++++++++++++++++ 2 files changed, 229 insertions(+), 229 deletions(-) create mode 100644 lib/gateway/docker-compose.gateway.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index c8d29691..c05c7568 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,6 +8,7 @@ include: - lib/drop-box/docker-compose.drop-box.yaml - lib/drs/docker-compose.drs.yaml - lib/event-relay/docker-compose.event-relay.yaml + - lib/gateway/docker-compose.gateway.yaml # Gateway/ingress; in production **only this** should bind host ports. - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml - lib/notification/docker-compose.notification.yaml @@ -18,232 +19,3 @@ include: - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory - -services: - gateway: - image: ${BENTOV2_GATEWAY_IMAGE}:${BENTOV2_GATEWAY_VERSION} - container_name: ${BENTOV2_GATEWAY_CONTAINER_NAME} - environment: - - BENTO_DEBUG=false - - - BENTO_AUTHZ_SERVICE_URL=${BENTO_AUTHZ_SERVICE_URL} - - BENTOV2_SESSION_SECRET - - - BENTOV2_AUTH_DOMAIN - - BENTOV2_DOMAIN - - BENTOV2_PORTAL_DOMAIN - - BENTOV2_CBIOPORTAL_DOMAIN - - BENTO_DOMAIN_REDIRECT - - # Feature flags: - - BENTOV2_USE_EXTERNAL_IDP - - BENTOV2_USE_BENTO_PUBLIC - - BENTO_USE_DOMAIN_REDIRECT - - BENTO_GATEWAY_USE_TLS - - BENTO_BEACON_ENABLED - - BENTO_CBIOPORTAL_ENABLED - - BENTO_GOHAN_ENABLED - - - BENTOV2_GATEWAY_CONTAINER_NAME - - - BENTOV2_GATEWAY_INTERNAL_CERTS_DIR - - BENTOV2_GATEWAY_INTERNAL_PORTAL_FULLCHAIN_RELATIVE_PATH - - BENTOV2_GATEWAY_INTERNAL_PORTAL_PRIVKEY_RELATIVE_PATH - - BENTOV2_GATEWAY_INTERNAL_FULLCHAIN_RELATIVE_PATH - - BENTOV2_GATEWAY_INTERNAL_PRIVKEY_RELATIVE_PATH - - BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_FULLCHAIN_RELATIVE_PATH - - BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_PRIVKEY_RELATIVE_PATH - - BENTO_GATEWAY_INTERNAL_REDIRECT_FULLCHAIN_RELATIVE_PATH - - BENTO_GATEWAY_INTERNAL_REDIRECT_PRIVKEY_RELATIVE_PATH - - - BENTOV2_GATEWAY_PUBLIC_ALLOW_FRAME_DOMAINS - - - BENTOV2_AUTH_CONTAINER_NAME - - BENTOV2_AUTH_INTERNAL_PORT - - BENTOV2_AUTH_INTERNAL_PORT_PLAIN_HTTP - - - BENTO_AUTHZ_CONTAINER_NAME - - BENTO_AUTHZ_INTERNAL_PORT - - BENTOV2_DROP_BOX_CONTAINER_NAME - - BENTOV2_DROP_BOX_INTERNAL_PORT - - BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME - - BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT - - BENTOV2_KATSU_CONTAINER_NAME - - BENTOV2_KATSU_INTERNAL_PORT - - BENTOV2_KATSU_DB_CONTAINER_NAME - - BENTOV2_KATSU_DB_INTERNAL_PORT - - BENTOV2_DRS_CONTAINER_NAME - - BENTOV2_DRS_INTERNAL_PORT - - BENTOV2_NOTIFICATION_CONTAINER_NAME - - BENTOV2_NOTIFICATION_INTERNAL_PORT - - BENTOV2_AGGREGATION_CONTAINER_NAME - - BENTOV2_AGGREGATION_INTERNAL_PORT - - BENTOV2_EVENT_RELAY_CONTAINER_NAME - - BENTOV2_EVENT_RELAY_INTERNAL_PORT - - BENTO_REFERENCE_CONTAINER_NAME - - BENTO_REFERENCE_INTERNAL_PORT - - BENTOV2_WES_CONTAINER_NAME - - BENTOV2_WES_INTERNAL_PORT - - BENTOV2_WEB_CONTAINER_NAME - - BENTOV2_WEB_INTERNAL_PORT - - BENTOV2_GOHAN_API_CONTAINER_NAME - - BENTOV2_GOHAN_API_INTERNAL_PORT - - BENTOV2_GOHAN_ES_CONTAINER_NAME - - BENTOV2_GOHAN_ES_INTERNAL_PORT_1 - - BENTOV2_GOHAN_ES_INTERNAL_PORT_2 - - BENTO_PUBLIC_CONTAINER_NAME - - BENTO_PUBLIC_INTERNAL_PORT - - BENTO_BEACON_CONTAINER_NAME - - BENTO_BEACON_INTERNAL_PORT - - BENTO_CBIOPORTAL_CONTAINER_NAME - - BENTO_CBIOPORTAL_INTERNAL_PORT - networks: - aggregation-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - auth-net: - authz-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - beacon-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - cbioportal-net: # No aliases needed here; container for sure does not contact the gateway - drop-box-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - drs-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - event-relay-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - gohan-api-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - katsu-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - notification-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - public-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - reference-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - service-registry-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - web-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - wes-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - ports: - - "80:80" - - "443:443" - expose: - - "80" - - "443" - - "444" - volumes: - - ${BENTOV2_GATEWAY_CERTS_DIR}:${BENTOV2_GATEWAY_INTERNAL_CERTS_DIR}:ro - - ${PWD}/lib/gateway/services:/gateway/services:ro - - ${PWD}/lib/gateway/public_services:/gateway/public_services:ro - mem_limit: ${BENTOV2_GATEWAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 - cpus: ${BENTOV2_GATEWAY_CPUS} - cpu_shares: 512 - healthcheck: - test: [ - "CMD", - "curl", "-k", - "--resolve", "${BENTOV2_PORTAL_DOMAIN}:443:127.0.0.1", - "https://${BENTOV2_PORTAL_DOMAIN}" - ] - timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} - interval: ${BENTOV2_HEALTHCHECK_INTERVAL} - - -# Lots of networks! Here, we have more or less one network per service, plus a few ones for databases -# (which don't even get exposed to the gateway) and some other extras. -# -# These are created by running ./bentoctl.bash init-docker ; -# the cBioPortal ones are only created if cBioPortal is enabled. -# -# The rationale behind this multi-network setup is that each service should, by default, not be able to talk to anything -# except what it *requires*. Databases should only be able to talk to the service which needs them and no one else, not -# even the gateway, since a database should never be publicly accessible. -# -# This is also sort of a quasi-self-documenting setup; if you only put services which need to talk together in the same -# network, you implicitly document the connections between said services. -# -# A concrete example: for the network katsu-net, the 'requirement for membership' is -# 'who needs to talk to katsu, without going through standard gateway/authentication procedures'? -# In this fashion, any other service put into katsu-net must have an important reason! - -networks: - aggregation-net: - external: true - name: ${BENTO_AGGREGATION_NETWORK} - auth-net: - external: true - name: ${BENTO_AUTH_NETWORK} - authz-net: - external: true - name: ${BENTO_AUTHZ_NETWORK} - beacon-net: - external: true - name: ${BENTO_BEACON_NETWORK} - cbioportal-net: - external: true - name: ${BENTO_CBIOPORTAL_NETWORK} - drop-box-net: - external: true - name: ${BENTO_DROP_BOX_NETWORK} - drs-net: - external: true - name: ${BENTO_DRS_NETWORK} - event-relay-net: - external: true - name: ${BENTO_EVENT_RELAY_NETWORK} - gohan-api-net: - external: true - name: ${BENTO_GOHAN_API_NETWORK} - katsu-net: - external: true - name: ${BENTO_KATSU_NETWORK} - notification-net: - external: true - name: ${BENTO_NOTIFICATION_NETWORK} - public-net: - external: true - name: ${BENTO_PUBLIC_NETWORK} - reference-net: - external: true - name: ${BENTO_REFERENCE_NETWORK} - service-registry-net: - external: true - name: ${BENTO_SERVICE_REGISTRY_NETWORK} - web-net: - external: true - name: ${BENTO_WEB_NETWORK} - wes-net: - external: true - name: ${BENTO_WES_NETWORK} diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml new file mode 100644 index 00000000..012d678b --- /dev/null +++ b/lib/gateway/docker-compose.gateway.yaml @@ -0,0 +1,228 @@ +services: + gateway: + image: ${BENTOV2_GATEWAY_IMAGE}:${BENTOV2_GATEWAY_VERSION} + container_name: ${BENTOV2_GATEWAY_CONTAINER_NAME} + environment: + - BENTO_DEBUG=false + + - BENTO_AUTHZ_SERVICE_URL=${BENTO_AUTHZ_SERVICE_URL} + - BENTOV2_SESSION_SECRET + + - BENTOV2_AUTH_DOMAIN + - BENTOV2_DOMAIN + - BENTOV2_PORTAL_DOMAIN + - BENTOV2_CBIOPORTAL_DOMAIN + - BENTO_DOMAIN_REDIRECT + + # Feature flags: + - BENTOV2_USE_EXTERNAL_IDP + - BENTOV2_USE_BENTO_PUBLIC + - BENTO_USE_DOMAIN_REDIRECT + - BENTO_GATEWAY_USE_TLS + - BENTO_BEACON_ENABLED + - BENTO_CBIOPORTAL_ENABLED + - BENTO_GOHAN_ENABLED + + - BENTOV2_GATEWAY_CONTAINER_NAME + + - BENTOV2_GATEWAY_INTERNAL_CERTS_DIR + - BENTOV2_GATEWAY_INTERNAL_PORTAL_FULLCHAIN_RELATIVE_PATH + - BENTOV2_GATEWAY_INTERNAL_PORTAL_PRIVKEY_RELATIVE_PATH + - BENTOV2_GATEWAY_INTERNAL_FULLCHAIN_RELATIVE_PATH + - BENTOV2_GATEWAY_INTERNAL_PRIVKEY_RELATIVE_PATH + - BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_FULLCHAIN_RELATIVE_PATH + - BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_PRIVKEY_RELATIVE_PATH + - BENTO_GATEWAY_INTERNAL_REDIRECT_FULLCHAIN_RELATIVE_PATH + - BENTO_GATEWAY_INTERNAL_REDIRECT_PRIVKEY_RELATIVE_PATH + + - BENTOV2_GATEWAY_PUBLIC_ALLOW_FRAME_DOMAINS + + - BENTOV2_AUTH_CONTAINER_NAME + - BENTOV2_AUTH_INTERNAL_PORT + - BENTOV2_AUTH_INTERNAL_PORT_PLAIN_HTTP + + - BENTO_AUTHZ_CONTAINER_NAME + - BENTO_AUTHZ_INTERNAL_PORT + - BENTOV2_DROP_BOX_CONTAINER_NAME + - BENTOV2_DROP_BOX_INTERNAL_PORT + - BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME + - BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT + - BENTOV2_KATSU_CONTAINER_NAME + - BENTOV2_KATSU_INTERNAL_PORT + - BENTOV2_KATSU_DB_CONTAINER_NAME + - BENTOV2_KATSU_DB_INTERNAL_PORT + - BENTOV2_DRS_CONTAINER_NAME + - BENTOV2_DRS_INTERNAL_PORT + - BENTOV2_NOTIFICATION_CONTAINER_NAME + - BENTOV2_NOTIFICATION_INTERNAL_PORT + - BENTOV2_AGGREGATION_CONTAINER_NAME + - BENTOV2_AGGREGATION_INTERNAL_PORT + - BENTOV2_EVENT_RELAY_CONTAINER_NAME + - BENTOV2_EVENT_RELAY_INTERNAL_PORT + - BENTO_REFERENCE_CONTAINER_NAME + - BENTO_REFERENCE_INTERNAL_PORT + - BENTOV2_WES_CONTAINER_NAME + - BENTOV2_WES_INTERNAL_PORT + - BENTOV2_WEB_CONTAINER_NAME + - BENTOV2_WEB_INTERNAL_PORT + - BENTOV2_GOHAN_API_CONTAINER_NAME + - BENTOV2_GOHAN_API_INTERNAL_PORT + - BENTOV2_GOHAN_ES_CONTAINER_NAME + - BENTOV2_GOHAN_ES_INTERNAL_PORT_1 + - BENTOV2_GOHAN_ES_INTERNAL_PORT_2 + - BENTO_PUBLIC_CONTAINER_NAME + - BENTO_PUBLIC_INTERNAL_PORT + - BENTO_BEACON_CONTAINER_NAME + - BENTO_BEACON_INTERNAL_PORT + - BENTO_CBIOPORTAL_CONTAINER_NAME + - BENTO_CBIOPORTAL_INTERNAL_PORT + networks: + aggregation-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + auth-net: + authz-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + beacon-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + cbioportal-net: # No aliases needed here; container for sure does not contact the gateway + drop-box-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + drs-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + event-relay-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + gohan-api-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + katsu-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + notification-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + public-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + reference-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + service-registry-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + web-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + wes-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + ports: + - "80:80" + - "443:443" + expose: + - "80" + - "443" + - "444" + volumes: + - ${BENTOV2_GATEWAY_CERTS_DIR}:${BENTOV2_GATEWAY_INTERNAL_CERTS_DIR}:ro + - ./lib/gateway/services:/gateway/services:ro + - ./lib/gateway/public_services:/gateway/public_services:ro + mem_limit: ${BENTOV2_GATEWAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 + cpus: ${BENTOV2_GATEWAY_CPUS} + cpu_shares: 512 + healthcheck: + test: [ + "CMD", + "curl", "-k", + "--resolve", "${BENTOV2_PORTAL_DOMAIN}:443:127.0.0.1", + "https://${BENTOV2_PORTAL_DOMAIN}" + ] + timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} + interval: ${BENTOV2_HEALTHCHECK_INTERVAL} + + +# Lots of networks! Here, we have more or less one network per service, plus a few ones for databases +# (which don't even get exposed to the gateway) and some other extras. +# +# These are created by running ./bentoctl.bash init-docker ; +# the cBioPortal ones are only created if cBioPortal is enabled. +# +# The rationale behind this multi-network setup is that each service should, by default, not be able to talk to anything +# except what it *requires*. Databases should only be able to talk to the service which needs them and no one else, not +# even the gateway, since a database should never be publicly accessible. +# +# This is also sort of a quasi-self-documenting setup; if you only put services which need to talk together in the same +# network, you implicitly document the connections between said services. +# +# A concrete example: for the network katsu-net, the 'requirement for membership' is +# 'who needs to talk to katsu, without going through standard gateway/authentication procedures'? +# In this fashion, any other service put into katsu-net must have an important reason! + +networks: + aggregation-net: + external: true + name: ${BENTO_AGGREGATION_NETWORK} + auth-net: + external: true + name: ${BENTO_AUTH_NETWORK} + authz-net: + external: true + name: ${BENTO_AUTHZ_NETWORK} + beacon-net: + external: true + name: ${BENTO_BEACON_NETWORK} + cbioportal-net: + external: true + name: ${BENTO_CBIOPORTAL_NETWORK} + drop-box-net: + external: true + name: ${BENTO_DROP_BOX_NETWORK} + drs-net: + external: true + name: ${BENTO_DRS_NETWORK} + event-relay-net: + external: true + name: ${BENTO_EVENT_RELAY_NETWORK} + gohan-api-net: + external: true + name: ${BENTO_GOHAN_API_NETWORK} + katsu-net: + external: true + name: ${BENTO_KATSU_NETWORK} + notification-net: + external: true + name: ${BENTO_NOTIFICATION_NETWORK} + public-net: + external: true + name: ${BENTO_PUBLIC_NETWORK} + reference-net: + external: true + name: ${BENTO_REFERENCE_NETWORK} + service-registry-net: + external: true + name: ${BENTO_SERVICE_REGISTRY_NETWORK} + web-net: + external: true + name: ${BENTO_WEB_NETWORK} + wes-net: + external: true + name: ${BENTO_WES_NETWORK} From 3d749ba19891e1353b25de445330e3ff225aef05 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 13 Mar 2024 10:53:31 -0400 Subject: [PATCH 024/130] docs: comment main compose file --- docker-compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c05c7568..fc5357d2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,8 @@ include: - - path: + - path: # By including all compose files in a single include.path[], they will get merged together. + # ===== Gateway/ingress; in production, **ONLY THIS** should bind host ports ===== + - lib/gateway/docker-compose.gateway.yaml + # ===== Services ================================================================= - lib/aggregation/docker-compose.aggregation.yaml - lib/auth/docker-compose.auth.yaml # Optional feature; controlled by a compose profile - lib/authz/docker-compose.authz.yaml @@ -8,7 +11,6 @@ include: - lib/drop-box/docker-compose.drop-box.yaml - lib/drs/docker-compose.drs.yaml - lib/event-relay/docker-compose.event-relay.yaml - - lib/gateway/docker-compose.gateway.yaml # Gateway/ingress; in production **only this** should bind host ports. - lib/gohan/docker-compose.gohan.yaml # Optional feature; controlled by a compose profile - lib/katsu/docker-compose.katsu.yaml - lib/notification/docker-compose.notification.yaml From 64dca04e820155e315b8ff4f7e9bd6bc41e22e0f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 13 Mar 2024 10:53:49 -0400 Subject: [PATCH 025/130] docs: lint --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index fc5357d2..05c9e908 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,8 @@ include: - path: # By including all compose files in a single include.path[], they will get merged together. - # ===== Gateway/ingress; in production, **ONLY THIS** should bind host ports ===== + # ===== Gateway/ingress; in production, **ONLY THIS** should bind host ports ===================================== - lib/gateway/docker-compose.gateway.yaml - # ===== Services ================================================================= + # ===== Services ================================================================================================= - lib/aggregation/docker-compose.aggregation.yaml - lib/auth/docker-compose.auth.yaml # Optional feature; controlled by a compose profile - lib/authz/docker-compose.authz.yaml From 107cf713e5ca07fc6496a5a3927a7f005a9b9a62 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 11 Mar 2024 15:13:35 -0400 Subject: [PATCH 026/130] docs: update migration guide --- docs/migrating_to_16.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index 283c9377..76fb1a82 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -1,20 +1,18 @@ # Migrating to Bento v16 -Bento v16 may now use 2 data directories for better resource usage in production. +Key points: + +* Bento v16 may now use 2 data directories for better resource usage in production. + * The environment variable `BENTOV2_ROOT_DATA_DIR` was replaced with: + * `BENTO_FAST_DATA_DIR` for SSD mounts + * `BENTO_SLOW_DATA_DIR` for HDD mounts + * By default, `Drop-Box` and `DRS` are now configured to use `BENTO_SLOW_DATA_DIR` as their data directory. + All other services use `BENTO_FAST_DATA_DIR`. + * If needed, the default service specific volume directories defined in [bento.env](../etc/bento.env) can be overriden in your + `local.env`. +* The WES schema has been updated, so the run database must be cleared. -The environment variable `BENTOV2_ROOT_DATA_DIR` was replaced with: -* `BENTO_FAST_DATA_DIR` for SSD mounts -* `BENTO_SLOW_DATA_DIR` for HDD mounts - - -By default, `Drop-Box` and `DRS` are now configured to use `BENTO_SLOW_DATA_DIR` as their data directory. - -All other services use `BENTO_FAST_DATA_DIR`. - -If needed, the default service specific volume directories defined in [bento.env](../etc/bento.env) can be overriden -in your `local.env`. - ## 1. Pre-update data config Before updating, perform the following steps: From 60e6f6bd033d3eb9224d7bbfeeebfad9916eb1f6 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 14 Mar 2024 12:55:26 -0400 Subject: [PATCH 027/130] set WES to edge and configure for artifact storage --- etc/bento.env | 2 +- lib/wes/docker-compose.wes.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index d7342dde..9fba62ea 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -200,7 +200,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=0.12.8 +BENTOV2_WES_VERSION=edge BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net diff --git a/lib/wes/docker-compose.wes.yaml b/lib/wes/docker-compose.wes.yaml index cd3363a6..4c155ac5 100644 --- a/lib/wes/docker-compose.wes.yaml +++ b/lib/wes/docker-compose.wes.yaml @@ -13,12 +13,12 @@ services: - CELERY_DEBUG=false - BENTO_EVENT_REDIS_URL=redis://${BENTOV2_REDIS_CONTAINER_NAME}:${BENTOV2_REDIS_INTERNAL_PORT} - - DATABASE=/wes/data/bento_wes.db - SERVICE_ID= + - SERVICE_DATA=/wes/data + - DATABASE=/wes/data/bento_wes.db - SERVICE_TEMP=/wes/tmp - SERVICE_BASE_URL=${BENTOV2_PUBLIC_URL}/api/wes/ - - WOM_TOOL_LOCATION= - #/wes/womtool.jar + - WOM_TOOL_LOCATION=/womtool.jar # Allow-list (comma-separated) for hosts that workflow files can be downloaded # from - prevents possibly insecure WDLs from being run - WORKFLOW_HOST_ALLOW_LIST=${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT},${BENTOV2_DOMAIN},${BENTOV2_PORTAL_DOMAIN},${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT} From 5a659dc321f59d8e171373c5eed6834531105643 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 14 Mar 2024 16:21:40 -0400 Subject: [PATCH 028/130] set service-registry to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9fba62ea..5bfffe77 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -123,7 +123,7 @@ BENTOV2_DROP_BOX_CPUS=3 # Service Registry BENTOV2_SERVICE_REGISTRY_IMAGE=ghcr.io/bento-platform/bento_service_registry -BENTOV2_SERVICE_REGISTRY_VERSION=1.3.5 +BENTOV2_SERVICE_REGISTRY_VERSION=edge BENTOV2_SERVICE_REGISTRY_VERSION_DEV=${BENTOV2_SERVICE_REGISTRY_VERSION}-dev BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME=${BENTOV2_PREFIX}-service-registry BENTO_SERVICE_REGISTRY_NETWORK=${BENTOV2_PREFIX}-service-registry-net From 0315ad9288a6e9aa6eb3efe204ecdf725d5b76cf Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Thu, 14 Mar 2024 16:54:42 -0400 Subject: [PATCH 029/130] add cors config to katsu --- etc/bento.env | 6 ++++-- lib/gateway/services/katsu.conf.tpl | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index cbed2860..1677bc67 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,8 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=0.10.0 +# BENTOV2_GATEWAY_VERSION=0.10.0 +BENTOV2_GATEWAY_VERSION=pr-12 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway @@ -374,7 +375,8 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=0.17.3 +# BENTO_PUBLIC_VERSION=0.17.3 +BENTO_PUBLIC_VERSION=pr-150 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net diff --git a/lib/gateway/services/katsu.conf.tpl b/lib/gateway/services/katsu.conf.tpl index 1a7bb148..81e47553 100644 --- a/lib/gateway/services/katsu.conf.tpl +++ b/lib/gateway/services/katsu.conf.tpl @@ -11,6 +11,9 @@ location /api/metadata/ { return 400; proxy_pass http://${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_INTERNAL_PORT}$uri; + # CORS + include /usr/local/openresty/nginx/conf/cors.conf; + # Errors error_log /var/log/bentov2_metadata_errors.log; -} \ No newline at end of file +} From 1b81589603e36134edd82eaccb13362c3bad6c9b Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 15 Mar 2024 11:26:01 -0400 Subject: [PATCH 030/130] add cbioportal client creation to init-auth --- docs/installation.md | 4 +++ etc/bento_deploy.env | 4 +++ etc/bento_dev.env | 4 +++ etc/default_config.env | 5 +++- py_bentoctl/auth_helper.py | 58 +++++++++++++++++++++++++++++++------- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 6dde6cca..11938cea 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -80,6 +80,10 @@ BENTOV2_AUTH_ADMIN_PASSWORD=admin # !!! obviously for dev only !!! BENTOV2_AUTH_TEST_USER=user BENTOV2_AUTH_TEST_PASSWORD=user # !!! obviously for dev only !!! +# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_CBIOPORTAL_CLIENT_ID=cbioportal +BENTO_CBIOPORTAL_CLIENT_SECRET= + # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index dae8fbe1..928fc6c8 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -45,6 +45,10 @@ BENTOV2_AUTH_TEST_PASSWORD= BENTO_AUTH_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! BENTO_AUTHZ_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! +# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_CBIOPORTAL_CLIENT_ID=cbioportal +BENTO_CBIOPORTAL_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! + # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 7da8e079..c47b7351 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -45,6 +45,10 @@ BENTOV2_AUTH_ADMIN_PASSWORD= BENTOV2_AUTH_TEST_USER= BENTOV2_AUTH_TEST_PASSWORD= +# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM +BENTO_CBIOPORTAL_CLIENT_ID=cbioportal +BENTO_CBIOPORTAL_CLIENT_SECRET= + # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/etc/default_config.env b/etc/default_config.env index 69a7592d..4f65fa15 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -29,7 +29,7 @@ BENTOV2_PREFIX=bentov2 # Previously BENTOV2_ROOT_DATA_DIR, now split for SSD/HDD optimization in prod # Root data directories on host. Bind volumes for services will be put # here as subdirectories and mounted into the relevant containers. -BENTO_FAST_DATA_DIR=${PWD}/data +BENTO_FAST_DATA_DIR=${PWD}/data BENTO_SLOW_DATA_DIR=${PWD}/data # Gateway/domains ----------------------------------------------------- @@ -73,6 +73,9 @@ BENTOV2_AUTH_TEST_PASSWORD= # - Auth (Keycloak) DB credentials BENTO_AUTH_DB_PASSWORD= BENTO_AUTHZ_DB_PASSWORD= +# - cBioPortal Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM +BENTO_CBIOPORTAL_CLIENT_ID=cbioportal +BENTO_CBIOPORTAL_CLIENT_SECRET= # - WES Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 3c5716cd..1b577953 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -36,6 +36,8 @@ AUTH_TEST_PASSWORD = os.getenv("BENTOV2_AUTH_TEST_PASSWORD") AUTH_CONTAINER_NAME = os.getenv("BENTOV2_AUTH_CONTAINER_NAME") +CBIOPORTAL_CLIENT_ID = os.getenv("BENTO_CBIOPORTAL_CLIENT_ID") + WES_CLIENT_ID = os.getenv("BENTO_WES_CLIENT_ID") WES_WORKFLOW_TIMEOUT = int(os.getenv("BENTOV2_WES_WORKFLOW_TIMEOUT")) @@ -151,6 +153,10 @@ def create_keycloak_client_or_exit( exit(1) +def get_keycloak_client_secret(client_id: str, token: str): + return keycloak_req(f"{KC_CLIENTS_ENDPOINT}/{client_id}/client-secret", bearer_token=token) + + def init_auth(docker_client: docker.DockerClient): check_auth_admin_user() @@ -204,8 +210,6 @@ def create_web_client_if_needed(token: str) -> None: if web_client_kc_id is not None: return - cbio_enabled = c.BENTO_FEATURE_CBIOPORTAL.enabled - # Create the Bento public/web client create_keycloak_client_or_exit( token, @@ -216,17 +220,47 @@ def create_web_client_if_needed(token: str) -> None: redirect_uris=[ f"{PUBLIC_URL}{AUTH_LOGIN_REDIRECT_PATH}", f"{PORTAL_PUBLIC_URL}{AUTH_LOGIN_REDIRECT_PATH}", - *((f"{CBIOPORTAL_URL}{AUTH_LOGIN_REDIRECT_PATH}",) if cbio_enabled else ()), - ], - web_origins=[ - f"{PUBLIC_URL}", - f"{PORTAL_PUBLIC_URL}", - *((f"{CBIOPORTAL_URL}",) if cbio_enabled else ()), ], + web_origins=[PUBLIC_URL, PORTAL_PUBLIC_URL], access_token_lifespan=900, # default access token lifespan: 15 minutes use_refresh_tokens=True, ) + def create_cbioportal_client_if_needed(token: str) -> None: + cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) + + if cbio_client_kc_id is None: + # Create the cBioportal client + create_keycloak_client_or_exit( + token, + CBIOPORTAL_CLIENT_ID, + standard_flow_enabled=True, + service_accounts_enabled=False, + public_client=False, + redirect_uris=[f"{CBIOPORTAL_URL}{AUTH_LOGIN_REDIRECT_PATH}"], + web_origins=[CBIOPORTAL_URL], + access_token_lifespan=900, # 15 minutes + use_refresh_tokens=True, + ) + cbio_client_kc_id = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) + + # Fetch and print secret + + client_secret_res = get_keycloak_client_secret(cbio_client_kc_id, token) + + client_secret_data = client_secret_res.json() + if not client_secret_res.ok: + err(f" Failed to get client secret for {CBIOPORTAL_CLIENT_ID}; {client_secret_res.status_code} " + f"{client_secret_data}") + exit(1) + + client_secret = client_secret_data["value"] + cprint( + f" Please set BENTO_CBIOPORTAL_CLIENT_SECRET to {client_secret} in local.env and restart the " + f"gateway", + attrs=["bold"], + ) + def create_wes_client_if_needed(token: str) -> None: wes_client_kc_id: Optional[str] = fetch_existing_client_id(token, WES_CLIENT_ID) @@ -247,8 +281,7 @@ def create_wes_client_if_needed(token: str) -> None: # Fetch and print secret - client_secret_res = keycloak_req( - f"{KC_CLIENTS_ENDPOINT}/{wes_client_kc_id}/client-secret", bearer_token=token) + client_secret_res = get_keycloak_client_secret(wes_client_kc_id, token) client_secret_data = client_secret_res.json() if not client_secret_res.ok: @@ -330,6 +363,11 @@ def success(): create_web_client_if_needed(access_token) success() + if c.BENTO_FEATURE_CBIOPORTAL.enabled: + info(f" Creating cBioPortal client: {CBIOPORTAL_CLIENT_ID}") + create_cbioportal_client_if_needed(access_token) + success() + info(f" Creating WES client: {WES_CLIENT_ID}") create_wes_client_if_needed(access_token) success() From e106562cd8b2720a909faacb40de7d5110073920 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 15 Mar 2024 14:49:26 -0400 Subject: [PATCH 031/130] chore: remove unneeded network aliases for gateway --- lib/gateway/docker-compose.gateway.yaml | 74 ++++++------------------- 1 file changed, 16 insertions(+), 58 deletions(-) diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index 012d678b..f493dab4 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -77,64 +77,22 @@ services: - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT networks: - aggregation-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - auth-net: - authz-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - beacon-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - cbioportal-net: # No aliases needed here; container for sure does not contact the gateway - drop-box-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - drs-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - event-relay-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - gohan-api-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - katsu-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - notification-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - public-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - reference-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - service-registry-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - web-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} - wes-net: - aliases: - - ${BENTOV2_DOMAIN} - - ${BENTOV2_PORTAL_DOMAIN} + - aggregation-net + - auth-net + - authz-net + - beacon-net + - cbioportal-net + - drop-box-net + - drs-net + - event-relay-net + - gohan-api-net + - katsu-net + - notification-net + - public-net + - reference-net + - service-registry-net + - web-net + - wes-net ports: - "80:80" - "443:443" From 2c440538ae812e0b145036c1a67382af4461eede Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 19 Mar 2024 10:11:03 -0400 Subject: [PATCH 032/130] set event relay to 3.1.1 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 5bfffe77..a67966a3 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -165,7 +165,7 @@ BENTOV2_AGGREGATION_CPUS=2 # Event-Relay BENTOV2_EVENT_RELAY_IMAGE=ghcr.io/bento-platform/bento_event_relay -BENTOV2_EVENT_RELAY_VERSION=3.1.0 +BENTOV2_EVENT_RELAY_VERSION=3.1.1 BENTOV2_EVENT_RELAY_VERSION_DEV=${BENTOV2_EVENT_RELAY_VERSION}-dev BENTOV2_EVENT_RELAY_CONTAINER_NAME=${BENTOV2_PREFIX}-event-relay BENTO_EVENT_RELAY_NETWORK=${BENTOV2_PREFIX}-event-relay-net From 8020a4fbf481fa225a721b0b83babf3dee86b1c1 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 19 Mar 2024 10:11:48 -0400 Subject: [PATCH 033/130] set authz to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index a67966a3..b08edd48 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -77,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net" # - Authz service BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service -BENTO_AUTHZ_VERSION=0.7.0 +BENTO_AUTHZ_VERSION=edge BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net From a7da23e1227a1a7fb943777e213e7f90b8110f15 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 19 Mar 2024 10:13:50 -0400 Subject: [PATCH 034/130] update bento keycloak dist to 2024-02-16 (v23) --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index b08edd48..3a9a7343 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -44,7 +44,7 @@ BENTOV2_GATEWAY_CPUS=2 # - Keycloak IdP - 'auth' BENTOV2_AUTH_IMAGE=ghcr.io/bento-platform/bento_keycloak_dist -BENTOV2_AUTH_VERSION=2023.10.31 +BENTOV2_AUTH_VERSION=2024.02.16 BENTOV2_AUTH_CONTAINER_NAME=${BENTOV2_PREFIX}-auth BENTO_AUTH_NETWORK=${BENTOV2_PREFIX}-auth-net BENTOV2_AUTH_SERVICE_HOST=0.0.0.0 From cd27880f7bef843f0a89915e44b68d8e27f6d57d Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 19 Mar 2024 11:27:20 -0400 Subject: [PATCH 035/130] use egde versions for gateway and public --- etc/bento.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 50a8e81d..10a576a7 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=pr-12 +BENTOV2_GATEWAY_VERSION=edge BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway @@ -375,7 +375,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=pr-150 +BENTO_PUBLIC_VERSION=edge BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From 230f1d8cb0d39b925d436f72867e9a1f312e52a8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 20 Mar 2024 15:16:55 -0400 Subject: [PATCH 036/130] docs: update dev container launch.json example --- docs/dev-container.md | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/dev-container.md b/docs/dev-container.md index 84b7c384..797a7a45 100644 --- a/docs/dev-container.md +++ b/docs/dev-container.md @@ -60,20 +60,18 @@ The port configuration for the debugger is passed using the file `launch.json` located in the `.vscode/` folder of the workspace. ```json { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Attach Debugger", - "type": "python", - "request": "attach", - "port": 5678, - "host": "0.0.0.0" - }, - ] + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Attach Debugger", + "type": "debugpy", + "request": "attach", + "listen": { + "host": "0.0.0.0", + "port": 5678 + } + } + ] } ``` From 1336a4ab041b82f27b960010cdd98876ccbbb5b8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 20 Mar 2024 15:45:44 -0400 Subject: [PATCH 037/130] set beacon to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 10a576a7..f8e2724d 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -396,7 +396,7 @@ BENTO_PUBLIC_TRANSLATED=true BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=0.13.0 +BENTO_BEACON_VERSION=edge BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 From d5ada2bba9e17ca7901b18a41ed151bce4999d08 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 25 Mar 2024 10:51:22 -0400 Subject: [PATCH 038/130] set gateway to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 380645a8..5ceed94b 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=pr-11 +BENTOV2_GATEWAY_VERSION=edge BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway From 78a8f7b293efce2cfd91eeac7d306a7b3c6c9291 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 25 Mar 2024 12:44:21 -0400 Subject: [PATCH 039/130] revert some cbioportal auth stuff (not needed for RENATA) --- docs/installation.md | 4 ---- etc/bento_deploy.env | 4 ---- etc/bento_dev.env | 4 ---- py_bentoctl/auth_helper.py | 12 ++++++++---- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 11938cea..6dde6cca 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -80,10 +80,6 @@ BENTOV2_AUTH_ADMIN_PASSWORD=admin # !!! obviously for dev only !!! BENTOV2_AUTH_TEST_USER=user BENTOV2_AUTH_TEST_PASSWORD=user # !!! obviously for dev only !!! -# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM -BENTO_CBIOPORTAL_CLIENT_ID=cbioportal -BENTO_CBIOPORTAL_CLIENT_SECRET= - # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index 928fc6c8..dae8fbe1 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -45,10 +45,6 @@ BENTOV2_AUTH_TEST_PASSWORD= BENTO_AUTH_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! BENTO_AUTHZ_DB_PASSWORD= # TODO: SET ME WHEN DEPLOYING! -# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM -BENTO_CBIOPORTAL_CLIENT_ID=cbioportal -BENTO_CBIOPORTAL_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! - # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= # TODO: SET ME WHEN DEPLOYING! diff --git a/etc/bento_dev.env b/etc/bento_dev.env index c47b7351..7da8e079 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -45,10 +45,6 @@ BENTOV2_AUTH_ADMIN_PASSWORD= BENTOV2_AUTH_TEST_USER= BENTOV2_AUTH_TEST_PASSWORD= -# - cBioPortal Client ID/secret; client within BENTOV2_AUTH_REALM -BENTO_CBIOPORTAL_CLIENT_ID=cbioportal -BENTO_CBIOPORTAL_CLIENT_SECRET= - # - WES Client ID/secret; client within BENTOV2_AUTH_REALM BENTO_WES_CLIENT_ID=wes BENTO_WES_CLIENT_SECRET= diff --git a/py_bentoctl/auth_helper.py b/py_bentoctl/auth_helper.py index 1b577953..9d9b2437 100644 --- a/py_bentoctl/auth_helper.py +++ b/py_bentoctl/auth_helper.py @@ -226,6 +226,7 @@ def create_web_client_if_needed(token: str) -> None: use_refresh_tokens=True, ) + # noinspection PyUnusedLocal def create_cbioportal_client_if_needed(token: str) -> None: cbio_client_kc_id: Optional[str] = fetch_existing_client_id(token, CBIOPORTAL_CLIENT_ID) @@ -363,10 +364,13 @@ def success(): create_web_client_if_needed(access_token) success() - if c.BENTO_FEATURE_CBIOPORTAL.enabled: - info(f" Creating cBioPortal client: {CBIOPORTAL_CLIENT_ID}") - create_cbioportal_client_if_needed(access_token) - success() + # TODO: if cBioPortal ever needs auth implemented, re-enable this and set up Bento Gateway to handle cBioPortal + # client authorization. + # - David L, 2024-03-25 + # if c.BENTO_FEATURE_CBIOPORTAL.enabled: + # info(f" Creating cBioPortal client: {CBIOPORTAL_CLIENT_ID}") + # create_cbioportal_client_if_needed(access_token) + # success() info(f" Creating WES client: {WES_CLIENT_ID}") create_wes_client_if_needed(access_token) From 51e8978f9668c342e47e66f96aad2812cabc2636 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:07:41 -0400 Subject: [PATCH 040/130] set katsu to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 3f3d220c..43a4db69 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -273,7 +273,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=7.0.2 +BENTOV2_KATSU_VERSION=edge BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net From eee91679c10b948b25ee6fb74e7479afee1dcdfa Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:16:51 -0400 Subject: [PATCH 041/130] revert pwd inclusion for gateway conf volumes --- lib/gateway/docker-compose.gateway.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index f493dab4..83b9385e 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -102,8 +102,8 @@ services: - "444" volumes: - ${BENTOV2_GATEWAY_CERTS_DIR}:${BENTOV2_GATEWAY_INTERNAL_CERTS_DIR}:ro - - ./lib/gateway/services:/gateway/services:ro - - ./lib/gateway/public_services:/gateway/public_services:ro + - ${PWD}/lib/gateway/services:/gateway/services:ro + - ${PWD}/lib/gateway/public_services:/gateway/public_services:ro mem_limit: ${BENTOV2_GATEWAY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_GATEWAY_CPUS} cpu_shares: 512 From d5abbadec6abba66ee7d60cc0b56ac67ae5c1914 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:19:54 -0400 Subject: [PATCH 042/130] revert to PWD for mounting bento_services.json to SR --- lib/service-registry/docker-compose.service-registry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/service-registry/docker-compose.service-registry.yaml b/lib/service-registry/docker-compose.service-registry.yaml index a966b3ab..dd6812e9 100644 --- a/lib/service-registry/docker-compose.service-registry.yaml +++ b/lib/service-registry/docker-compose.service-registry.yaml @@ -35,7 +35,7 @@ services: expose: - ${BENTOV2_SERVICE_REGISTRY_INTERNAL_PORT} volumes: - - ./etc/bento_services.json:/bento_services.json:ro + - ${PWD}/etc/bento_services.json:/bento_services.json:ro mem_limit: ${BENTOV2_SERVICE_REGISTRY_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_SERVICE_REGISTRY_CPUS} cpu_shares: 256 From b7226b581679bc58df61f44747a9d6c554c016c6 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:21:28 -0400 Subject: [PATCH 043/130] fix katsu config.json mount --- lib/katsu/docker-compose.katsu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/katsu/docker-compose.katsu.yaml b/lib/katsu/docker-compose.katsu.yaml index 331c8816..e8da7e3f 100644 --- a/lib/katsu/docker-compose.katsu.yaml +++ b/lib/katsu/docker-compose.katsu.yaml @@ -42,7 +42,7 @@ services: # target: metadata_db_secret volumes: # Custom Katsu config - - ./lib/katsu/config.json:${BENTOV2_KATSU_INTERNAL_PROJECT_DIRECTORY}/chord_metadata_service/config.json:ro + - ${PWD}/lib/katsu/config.json:${BENTOV2_KATSU_INTERNAL_PROJECT_DIRECTORY}/chord_metadata_service/config.json:ro # Temp WES dir (read only mount, since we are just pulling files from it) - ${BENTOV2_KATSU_PROD_WES_VOL_TMP_DIR}:/wes/tmp:ro mem_limit: ${BENTOV2_KATSU_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 From 593e97ac01cbf3b65e8415a956d6d31f82b7acb2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:27:58 -0400 Subject: [PATCH 044/130] chore: remove cbioportal block in prod mode --- py_bentoctl/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index e4b589f8..196ede33 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -106,11 +106,6 @@ def __init__(self, enabled: bool, profile: str): BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public") BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect") -if not DEV_MODE and BENTO_FEATURE_CBIOPORTAL.enabled: - import sys - print("cBioPortal for production deployments is not finished.", file=sys.stderr) - exit(1) - BENTO_GIT_CLONE_HTTPS: bool = os.getenv("BENTO_GIT_CLONE_HTTPS", "0").lower().strip() in ("1", "true") From 5897528a2801fb6c43170ac4d878d08aec430640 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:28:11 -0400 Subject: [PATCH 045/130] lint --- py_bentoctl/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py_bentoctl/config.py b/py_bentoctl/config.py index 196ede33..be1c06dc 100644 --- a/py_bentoctl/config.py +++ b/py_bentoctl/config.py @@ -106,7 +106,6 @@ def __init__(self, enabled: bool, profile: str): BENTO_FEATURE_PUBLIC = BentoOptionalFeature(enabled=BENTOV2_USE_BENTO_PUBLIC, profile="public") BENTO_FEATURE_REDIRECT = BentoOptionalFeature(enabled=bool(BENTO_DOMAIN_REDIRECT), profile="redirect") - BENTO_GIT_CLONE_HTTPS: bool = os.getenv("BENTO_GIT_CLONE_HTTPS", "0").lower().strip() in ("1", "true") COMPOSE: Tuple[str, ...] = ("docker", "compose") From 23459ae9499a54f37e915111862c1af58dc1b241 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:43:32 -0400 Subject: [PATCH 046/130] fix: rm exposed cbioportal port in production --- docker-compose.dev.yaml | 4 ++++ lib/cbioportal/docker-compose.cbioportal.yaml | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 8cb0d634..7bf3f361 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -228,3 +228,7 @@ services: "http.cors.enabled": "true" "http.cors.allow-origin": "http://localhost:8081" "http.cors.allow-headers": X-Requested-With,Content-Type,Content-Length,Authorization + + cbioportal: + ports: + - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" diff --git a/lib/cbioportal/docker-compose.cbioportal.yaml b/lib/cbioportal/docker-compose.cbioportal.yaml index 01d7d29c..06251fdf 100644 --- a/lib/cbioportal/docker-compose.cbioportal.yaml +++ b/lib/cbioportal/docker-compose.cbioportal.yaml @@ -8,8 +8,6 @@ services: - cbioportal-net - cbioportal-db-net - cbioportal-session-net - ports: - - "${BENTO_CBIOPORTAL_EXTERNAL_PORT}:${BENTO_CBIOPORTAL_INTERNAL_PORT}" volumes: - ${BENTO_CBIOPORTAL_STUDY_DIR}:/study/ - ./lib/cbioportal/config/portal.properties:/cbioportal/portal.properties:ro From f624c2edf896203787b16723aee76ad57990980f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 15:57:22 -0400 Subject: [PATCH 047/130] use PWD for cbioportal volumes --- lib/cbioportal/docker-compose.cbioportal.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cbioportal/docker-compose.cbioportal.yaml b/lib/cbioportal/docker-compose.cbioportal.yaml index 06251fdf..a1d0fb1e 100644 --- a/lib/cbioportal/docker-compose.cbioportal.yaml +++ b/lib/cbioportal/docker-compose.cbioportal.yaml @@ -10,7 +10,7 @@ services: - cbioportal-session-net volumes: - ${BENTO_CBIOPORTAL_STUDY_DIR}:/study/ - - ./lib/cbioportal/config/portal.properties:/cbioportal/portal.properties:ro + - ${PWD}/lib/cbioportal/config/portal.properties:/cbioportal/portal.properties:ro depends_on: - cbioportal-db - cbioportal-session @@ -39,9 +39,9 @@ services: networks: - cbioportal-db-net volumes: - - ./lib/cbioportal/seed_data/cgds.sql:/docker-entrypoint-initdb.d/cgds.sql:ro - - ./lib/cbioportal/seed_data/seed-hg19.sql.gz:/docker-entrypoint-initdb.d/seed-hg19.sql.gz:ro - # - ./lib/cbioportal/seed_data/seed-hg38.sql.gz:/docker-entrypoint-initdb.d/seed-hg38.sql.gz:ro + - ${PWD}/lib/cbioportal/seed_data/cgds.sql:/docker-entrypoint-initdb.d/cgds.sql:ro + # - ./lib/cbioportal/seed_data/seed-hg19.sql.gz:/docker-entrypoint-initdb.d/seed-hg19.sql.gz:ro + - ${PWD}/lib/cbioportal/seed_data/seed-hg38.sql.gz:/docker-entrypoint-initdb.d/seed-hg38.sql.gz:ro - ${BENTO_CBIOPORTAL_DATABASE_DATA_DIR}:/var/lib/mysql profiles: - cbioportal From d8d57e3798daf21b9a36aa5f9f229318926f8346 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 16:18:38 -0400 Subject: [PATCH 048/130] fix(cbioportal): broken seed sql links --- py_bentoctl/feature_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py_bentoctl/feature_helpers.py b/py_bentoctl/feature_helpers.py index 09cc4134..fdd42785 100644 --- a/py_bentoctl/feature_helpers.py +++ b/py_bentoctl/feature_helpers.py @@ -17,9 +17,9 @@ f"/db-scripts/src/main/resources/cgds.sql" ) CBIOPORTAL_SEED_DB_HG19_URL = \ - "https://github.com/cBioPortal/datahub/raw/master/seedDB/seed-cbioportal_hg19_v2.12.12.sql.gz" + "https://github.com/cBioPortal/datahub/raw/master/seedDB/seed-cbioportal_hg19_v2.12.14.sql.gz" CBIOPORTAL_SEED_DB_HG38_URL = \ - "https://github.com/cBioPortal/datahub/raw/master/seedDB_hg38/seed-cbioportal_hg19_hg38_v2.12.12.sql.gz" + "https://github.com/cBioPortal/datahub/raw/master/seedDB/seed-cbioportal_hg19_hg38_v2.13.1.sql.gz" CBIOPORTAL_SEED_DATA_PATH = pathlib.Path(__file__).parent.parent / "lib" / "cbioportal" / "seed_data" CBIOPORTAL_SCHEMA_PATH = CBIOPORTAL_SEED_DATA_PATH / "cgds.sql" From f8bcbe9251c6ad60b5a865671f7c5a6539ee4e9f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 16:20:07 -0400 Subject: [PATCH 049/130] fix(cbioportal): non-changing links for seed db --- py_bentoctl/feature_helpers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/py_bentoctl/feature_helpers.py b/py_bentoctl/feature_helpers.py index fdd42785..727beb62 100644 --- a/py_bentoctl/feature_helpers.py +++ b/py_bentoctl/feature_helpers.py @@ -16,10 +16,12 @@ f"https://raw.githubusercontent.com/cBioPortal/cbioportal/v{CBIOPORTAL_VERSION}" f"/db-scripts/src/main/resources/cgds.sql" ) -CBIOPORTAL_SEED_DB_HG19_URL = \ - "https://github.com/cBioPortal/datahub/raw/master/seedDB/seed-cbioportal_hg19_v2.12.14.sql.gz" -CBIOPORTAL_SEED_DB_HG38_URL = \ - "https://github.com/cBioPortal/datahub/raw/master/seedDB/seed-cbioportal_hg19_hg38_v2.13.1.sql.gz" +CBIOPORTAL_SEED_DB_HG19_URL = ( + "https://github.com/cBioPortal/datahub/raw/ef7e21214a84f31393e3dd197ca6b78a1fc42698/seedDB/" + "seed-cbioportal_hg19_v2.12.14.sql.gz") +CBIOPORTAL_SEED_DB_HG38_URL = ( + "https://github.com/cBioPortal/datahub/raw/ef7e21214a84f31393e3dd197ca6b78a1fc42698/seedDB/" + "seed-cbioportal_hg19_hg38_v2.13.1.sql.gz") CBIOPORTAL_SEED_DATA_PATH = pathlib.Path(__file__).parent.parent / "lib" / "cbioportal" / "seed_data" CBIOPORTAL_SCHEMA_PATH = CBIOPORTAL_SEED_DATA_PATH / "cgds.sql" From 1cb6812a8baf62b089baa5a0985eae33c24ecfd5 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 26 Mar 2024 16:28:32 -0400 Subject: [PATCH 050/130] fix(cbioportal): hg19 seed db data download link --- py_bentoctl/feature_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_bentoctl/feature_helpers.py b/py_bentoctl/feature_helpers.py index 727beb62..dab41a52 100644 --- a/py_bentoctl/feature_helpers.py +++ b/py_bentoctl/feature_helpers.py @@ -17,7 +17,7 @@ f"/db-scripts/src/main/resources/cgds.sql" ) CBIOPORTAL_SEED_DB_HG19_URL = ( - "https://github.com/cBioPortal/datahub/raw/ef7e21214a84f31393e3dd197ca6b78a1fc42698/seedDB/" + "https://github.com/cBioPortal/datahub/raw/ef7e21214a84f31393e3dd197ca6b78a1fc42698/seedDB/seedDB_hg19_archive/" "seed-cbioportal_hg19_v2.12.14.sql.gz") CBIOPORTAL_SEED_DB_HG38_URL = ( "https://github.com/cBioPortal/datahub/raw/ef7e21214a84f31393e3dd197ca6b78a1fc42698/seedDB/" From b95b73c1f81d5169b77c7add0abe262beb1deb15 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Wed, 27 Mar 2024 09:07:46 -0400 Subject: [PATCH 051/130] add missing backslash --- docs/development.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.md b/docs/development.md index 09278079..c357bb1c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -72,7 +72,7 @@ git checkout [...] # Tag 1 => BENTOV2_WEB_IMAGE:BENTOV2_WEB_VERSION # Tag 2 => BENTOV2_WEB_IMAGE:BENTOV2_WEB_VERSION_DEV docker build -f dev.Dockerfile . \ - -t ghcr.io/bento-platform/bento_web:localonly + -t ghcr.io/bento-platform/bento_web:localonly \ -t ghcr.io/bento-platform/bento_web:localonly-dev # Back to root From 0d813f62b2927c7de9dd13017a61d9eaba6a22b2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 2 Apr 2024 11:13:14 -0400 Subject: [PATCH 052/130] fix: typo in cert renew script --- etc/scripts/renew_certs_and_redeploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/scripts/renew_certs_and_redeploy.sh b/etc/scripts/renew_certs_and_redeploy.sh index 860c31c0..a7750ac6 100755 --- a/etc/scripts/renew_certs_and_redeploy.sh +++ b/etc/scripts/renew_certs_and_redeploy.sh @@ -17,7 +17,7 @@ SERVICES=('gateway') # Auth needs cert renewal when internal only if [[ $BENTOV2_USE_EXTERNAL_IDP -eq 0 ]]; then - echo "Uses internal IDP, will attemps auth certs renewal." + echo "Uses internal IDP, will attempt auth certs renewal." # Append auth to list of services to renew SERVICES+=('auth') fi From 2361518835815e01758873a050d94e109d7f127b Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 2 Apr 2024 11:13:29 -0400 Subject: [PATCH 053/130] fix: missing reference-net aliases for gateway in dev compose --- docker-compose.dev.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 7bf3f361..730ca97f 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -70,6 +70,11 @@ services: - ${BENTOV2_DOMAIN} - ${BENTOV2_PORTAL_DOMAIN} - ${BENTOV2_AUTH_DOMAIN} + reference-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + - ${BENTOV2_AUTH_DOMAIN} service-registry-net: aliases: - ${BENTOV2_DOMAIN} From c35895a687eaeeb41de7d602b2697a669fba6e5a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 2 Apr 2024 15:48:12 -0400 Subject: [PATCH 054/130] chore: BENTO_PUBLIC_TRANSLATED should be set in local.env --- docs/installation.md | 5 +++++ etc/bento.env | 2 +- etc/bento_deploy.env | 5 +++++ etc/bento_dev.env | 5 +++++ etc/default_config.env | 5 +++++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index d207e730..3783462e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -44,6 +44,7 @@ BENTOV2_CBIOPORTAL_DOMAIN=cbioportal.${BENTOV2_DOMAIN} # --------------------------------------------------------------------- # Feature switches ---------------------------------------------------- + BENTOV2_USE_EXTERNAL_IDP=0 BENTOV2_USE_BENTO_PUBLIC=1 @@ -54,6 +55,10 @@ BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' + +# - Switch to enable French translation in Bento Public +BENTO_PUBLIC_TRANSLATED='true' + # --------------------------------------------------------------------- # Set this to a data storage location, optionally within the repo itself, like: /path-to-my-bentov2-repo/data diff --git a/etc/bento.env b/etc/bento.env index bf1a75bb..3a234f0e 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -390,7 +390,7 @@ BENTO_PUBLIC_KATSU_URL=http://${BENTOV2_KATSU_CONTAINER_NAME}:${BENTOV2_KATSU_IN BENTO_PUBLIC_WES_URL=http://${BENTOV2_WES_CONTAINER_NAME}:${BENTOV2_WES_INTERNAL_PORT} BENTO_PUBLIC_GOHAN_URL=http://${BENTOV2_GOHAN_API_CONTAINER_NAME}:${BENTOV2_GOHAN_API_INTERNAL_PORT} BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} -BENTO_PUBLIC_TRANSLATED=true +# BENTO_PUBLIC_TRANSLATED is set in default_config.env and can be overwritten in local.env # Beacon BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env index dae8fbe1..86a5c98a 100644 --- a/etc/bento_deploy.env +++ b/etc/bento_deploy.env @@ -1,6 +1,7 @@ MODE=prod # Feature switches start ---------------------------------------------- + BENTOV2_USE_EXTERNAL_IDP=0 BENTOV2_USE_BENTO_PUBLIC=1 @@ -11,6 +12,10 @@ BENTO_BEACON_ENABLED='false' # Set to true if using Beacon! BENTO_BEACON_UI_ENABLED='false' BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' + +# - Switch to enable French translation in Bento Public +BENTO_PUBLIC_TRANSLATED='true' + # Feature switches end ------------------------------------------------ # Prefix for standardized container names diff --git a/etc/bento_dev.env b/etc/bento_dev.env index 7da8e079..884d1d65 100644 --- a/etc/bento_dev.env +++ b/etc/bento_dev.env @@ -1,6 +1,7 @@ MODE=dev # Feature switches start ---------------------------------------------- + BENTOV2_USE_EXTERNAL_IDP=0 BENTOV2_USE_BENTO_PUBLIC=1 @@ -11,6 +12,10 @@ BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' + +# - Switch to enable French translation in Bento Public +BENTO_PUBLIC_TRANSLATED='true' + # Feature switches end ------------------------------------------------ # Previously BENTOV2_ROOT_DATA_DIR, now split for SSD/HDD optimization in prod diff --git a/etc/default_config.env b/etc/default_config.env index 4f65fa15..949f7309 100644 --- a/etc/default_config.env +++ b/etc/default_config.env @@ -6,6 +6,7 @@ BENTO_UID="$(id -u)" MODE=prod # Feature switches start ---------------------------------------------- + BENTOV2_USE_EXTERNAL_IDP=0 BENTOV2_USE_BENTO_PUBLIC=1 @@ -16,6 +17,10 @@ BENTO_BEACON_ENABLED='true' BENTO_BEACON_UI_ENABLED='true' BENTO_CBIOPORTAL_ENABLED='false' BENTO_GOHAN_ENABLED='true' + +# - Switch to enable French translation in Bento Public +BENTO_PUBLIC_TRANSLATED='true' + # Feature switches end ------------------------------------------------ # Whether to use HTTPS for cloning. Not recommended for development, but may be useful if From a3d7438fc5cae8ff3b49c23b4f03fb782d20ea6a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 3 Apr 2024 16:24:18 -0400 Subject: [PATCH 055/130] chore: simplify DRS volume dir env vars --- docker-compose.dev.yaml | 5 ----- etc/bento.env | 9 ++------- lib/drs/docker-compose.drs.yaml | 4 ++-- py_bentoctl/other_helpers.py | 2 +- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 730ca97f..96dddf5f 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -166,11 +166,6 @@ services: ports: - "${BENTOV2_DRS_EXTERNAL_PORT}:5000" - "${BENTOV2_DRS_DEBUGGER_EXTERNAL_PORT}:${BENTOV2_DRS_DEBUGGER_INTERNAL_PORT}" - volumes: - # Override main drs data directory - - ${BENTOV2_DRS_DEV_VOL_DIR}:/drs/bento_drs/data - - ${BENTOV2_DRS_DEV_WES_VOL_DIR}:/wes/tmp:ro - - ${BENTOV2_GOHAN_API_DRS_BRIDGE_HOST_DIR}:${BENTOV2_GOHAN_DRS_API_DRS_BRIDGE_DIR_CONTAINERIZED} beacon: environment: diff --git a/etc/bento.env b/etc/bento.env index 3a234f0e..0166ed62 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -238,13 +238,8 @@ BENTOV2_DRS_EXTERNAL_PORT=7007 BENTO_DRS_CONTAINER_DATA_VOLUME_DIR=/drs/bento_drs/data BENTO_DRS_CONTAINER_TMP_VOLUME_DIR=/drs/tmp -BENTOV2_DRS_PROD_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data -BENTOV2_DRS_PROD_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} -# ( assumes a relative wes path like ./lib/wes/... - adjust accordingly ) - -BENTOV2_DRS_DEV_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data -BENTOV2_DRS_DEV_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} - +BENTO_DRS_DATA_VOL_DIR=${BENTO_SLOW_DATA_DIR}/drs/data +BENTO_DRS_WES_VOL_DIR=${BENTOV2_WES_VOL_TMP_DIR} BENTO_DRS_TMP_VOL_DIR=${BENTO_FAST_DATA_DIR}/drs/tmp BENTOV2_DRS_DEBUGGER_INTERNAL_PORT=5682 diff --git a/lib/drs/docker-compose.drs.yaml b/lib/drs/docker-compose.drs.yaml index 6bf8063a..09c56b9d 100644 --- a/lib/drs/docker-compose.drs.yaml +++ b/lib/drs/docker-compose.drs.yaml @@ -18,9 +18,9 @@ services: expose: - ${BENTOV2_DRS_INTERNAL_PORT} volumes: - - ${BENTOV2_DRS_PROD_VOL_DIR}:${BENTO_DRS_CONTAINER_DATA_VOLUME_DIR} + - ${BENTO_DRS_DATA_VOL_DIR}:${BENTO_DRS_CONTAINER_DATA_VOLUME_DIR} - ${BENTO_DRS_TMP_VOL_DIR}:${BENTO_DRS_CONTAINER_TMP_VOLUME_DIR} - - ${BENTOV2_DRS_PROD_WES_VOL_DIR}:/wes/tmp:ro + - ${BENTO_DRS_WES_VOL_DIR}:/wes/tmp:ro - ${BENTOV2_GOHAN_API_DRS_BRIDGE_HOST_DIR}:${BENTOV2_GOHAN_DRS_API_DRS_BRIDGE_DIR_CONTAINERIZED} mem_limit: ${BENTOV2_DRS_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4 cpus: ${BENTOV2_DRS_CPUS} diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index d444ef71..2d00eb81 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -200,7 +200,7 @@ def init_dirs(): "root_fast": "BENTO_FAST_DATA_DIR", "root_slow": "BENTO_SLOW_DATA_DIR", "authz-db": "BENTO_AUTHZ_DB_VOL_DIR", - "drs-data": "BENTOV2_DRS_DEV_VOL_DIR" if c.DEV_MODE else "BENTOV2_DRS_PROD_VOL_DIR", + "drs-data": "BENTO_DRS_DATA_VOL_DIR", "drs-tmp": "BENTO_DRS_TMP_VOL_DIR", "drop-box": "BENTOV2_DROP_BOX_VOL_DIR", "gohan": "BENTOV2_GOHAN_DATA_ROOT", From 7d79a6df79222314e89d0dc33d152571c987dd79 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 3 Apr 2024 16:27:48 -0400 Subject: [PATCH 056/130] fix: mount DRS data volume into WES for vcf2maf --- lib/wes/docker-compose.wes.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/wes/docker-compose.wes.yaml b/lib/wes/docker-compose.wes.yaml index 4c155ac5..8699ec57 100644 --- a/lib/wes/docker-compose.wes.yaml +++ b/lib/wes/docker-compose.wes.yaml @@ -54,7 +54,10 @@ services: volumes: - ${BENTOV2_WES_VOL_DIR}:/wes/data - ${BENTOV2_WES_VOL_TMP_DIR}:/wes/tmp - - ${BENTOV2_WES_PROD_DROP_BOX_VOL_DIR}:/data:ro # Read only access to drop box contents via file path + # Read only access to drop box contents via file path: + - ${BENTOV2_WES_PROD_DROP_BOX_VOL_DIR}:/data:ro + # Read-only access to DRS objects via file path: + - ${BENTO_DRS_DATA_VOL_DIR}:${BENTO_DRS_CONTAINER_DATA_VOLUME_DIR}:ro networks: redis-net: From 3f932ee7a8cac2b3c0884a4d7c9431287602051b Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 9 Apr 2024 09:50:37 -0400 Subject: [PATCH 057/130] set wes to 0.13.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 0166ed62..e71b18d9 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -200,7 +200,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=edge +BENTOV2_WES_VERSION=0.13.0 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net From 38edac6c410f02d6893057ad5c684dafa6413474 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 9 Apr 2024 14:18:57 -0400 Subject: [PATCH 058/130] chore: default cbio to hg38 in portal.properties --- lib/cbioportal/config/portal.properties | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cbioportal/config/portal.properties b/lib/cbioportal/config/portal.properties index fd03421f..c6c0361c 100644 --- a/lib/cbioportal/config/portal.properties +++ b/lib/cbioportal/config/portal.properties @@ -262,9 +262,10 @@ disabled_tabs= priority_studies= # species and genomic information +# - Modified by David L 2024-04-09: default to hg38/GRCh38 species=human -ncbi.build=37 -ucsc.build=hg19 +ncbi.build=38 +ucsc.build=hg38 # default view in oncoprint (sample, patient (default)) oncoprint.defaultview=patient From 476d47c9f141b37dcdddce3daf37c5c8892f49ee Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 10 Apr 2024 12:39:07 -0400 Subject: [PATCH 059/130] chore: bump DRS to 0.16.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index e71b18d9..9ec0c3c3 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -227,7 +227,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800 # DRS BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs -BENTOV2_DRS_VERSION=0.15.3 +BENTOV2_DRS_VERSION=0.16.0 BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net From f5c9573aab4991209b500478e2293891e0409da8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 17 Apr 2024 13:28:18 -0400 Subject: [PATCH 060/130] set drs to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9ec0c3c3..a26a677e 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -227,7 +227,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800 # DRS BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs -BENTOV2_DRS_VERSION=0.16.0 +BENTOV2_DRS_VERSION=edge BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net From b1d16e51044dd778d7e996e48477c10bdde40a7c Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 22 Apr 2024 09:19:01 -0400 Subject: [PATCH 061/130] chore: update requirements --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2e45e076..f4559783 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ certifi==2024.2.2 cffi==1.16.0 charset-normalizer==3.3.2 -cryptography==42.0.4 -debugpy==1.8.0 +cryptography==42.0.5 +debugpy==1.8.1 docker==7.0.0 flake8==7.0.0 -idna==3.6 +idna==3.7 mccabe==0.7.0 -packaging==23.2 +packaging==24.0 pycodestyle==2.11.1 -pycparser==2.21 +pycparser==2.22 pyflakes==3.2.0 pyhumps==3.8.0 PyYAML==6.0.1 requests==2.31.0 termcolor==2.4.0 -tqdm==4.66.1 -urllib3==2.2.0 +tqdm==4.66.2 +urllib3==2.2.1 websocket-client==1.7.0 From 44826d5653750cd1a2291d2e601819c542b526c2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 22 Apr 2024 09:37:44 -0400 Subject: [PATCH 062/130] set service registry to 1.3.7 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index a26a677e..a6867c5f 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -123,7 +123,7 @@ BENTOV2_DROP_BOX_CPUS=3 # Service Registry BENTOV2_SERVICE_REGISTRY_IMAGE=ghcr.io/bento-platform/bento_service_registry -BENTOV2_SERVICE_REGISTRY_VERSION=edge +BENTOV2_SERVICE_REGISTRY_VERSION=1.3.7 BENTOV2_SERVICE_REGISTRY_VERSION_DEV=${BENTOV2_SERVICE_REGISTRY_VERSION}-dev BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME=${BENTOV2_PREFIX}-service-registry BENTO_SERVICE_REGISTRY_NETWORK=${BENTOV2_PREFIX}-service-registry-net From b3118e405f0d5d2e02f3c695ee88c16b3610c360 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 26 Apr 2024 10:22:59 -0400 Subject: [PATCH 063/130] set notification to 3.1.2 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index a6867c5f..fa7d0982 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -135,7 +135,7 @@ BENTOV2_SERVICE_REGISTRY_CPUS=1 # Notification BENTOV2_NOTIFICATION_IMAGE=ghcr.io/bento-platform/bento_notification_service -BENTOV2_NOTIFICATION_VERSION=3.1.1 +BENTOV2_NOTIFICATION_VERSION=3.1.2 BENTOV2_NOTIFICATION_VERSION_DEV=${BENTOV2_NOTIFICATION_VERSION}-dev BENTOV2_NOTIFICATION_CONTAINER_NAME=${BENTOV2_PREFIX}-notification BENTO_NOTIFICATION_NETWORK=${BENTOV2_PREFIX}-notification-net From 90ad8ae6e7e416f0189557bbb78f679846d4a7ec Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 29 Apr 2024 14:02:38 -0400 Subject: [PATCH 064/130] moved beacon config to etc/templates --- .../templates/beacon/beacon_config.example.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/beacon/config/beacon_config.json => etc/templates/beacon/beacon_config.example.json (100%) diff --git a/lib/beacon/config/beacon_config.json b/etc/templates/beacon/beacon_config.example.json similarity index 100% rename from lib/beacon/config/beacon_config.json rename to etc/templates/beacon/beacon_config.example.json From 4f86d9b5805b4340c659b6921bbad942f5b9c393 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 29 Apr 2024 14:05:49 -0400 Subject: [PATCH 065/130] added beacon_config.json to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5e624d19..c1ccf64b 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,9 @@ lib/web/bento_web lib/katsu/config.json +# ignored instance specific beacon config +lib/beacon/config/beacon_config.json + # bento_web lib/web/* !lib/web/.gitkeep From 4d0227730368d2031e22ecf9ad1bfee219cac286 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 29 Apr 2024 14:51:49 -0400 Subject: [PATCH 066/130] added option to copy the config files for katsu and beacon to their respective lib directories --- py_bentoctl/entry.py | 16 ++++++++++++++++ py_bentoctl/other_helpers.py | 23 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/py_bentoctl/entry.py b/py_bentoctl/entry.py index 3038c056..14c297f9 100644 --- a/py_bentoctl/entry.py +++ b/py_bentoctl/entry.py @@ -277,6 +277,21 @@ def exec(args): fh.init_cbioportal() +class InitConfig(SubCommand): + + @staticmethod + def add_args(sp): + sp.add_argument( + "service", type=str, choices=["katsu", "beacon"], help="Prepares services for deployment.") + sp.add_argument( + "--force", "-f", action="store_true", + help="Overwrites any existing config.") + + @staticmethod + def exec(args): + oh.init_config(args.service, args.force) + + class ConvertPhenopacket(SubCommand): @staticmethod @@ -330,6 +345,7 @@ def _add_subparser(arg: str, help_text: str, subcommand: Type[SubCommand], alias "init-all", "Initialize certs, directories, Docker networks, secrets, and web portals. DOES NOT initialize Keycloak.", InitAll) + _add_subparser("init-config", "Initialize configuration files for specific services.", InitConfig) # Feature-specific initialization commands _add_subparser("init-cbioportal", "Initialize cBioPortal if enabled", InitCBioPortal) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 2d00eb81..7152dbac 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -21,6 +21,7 @@ "init_docker", "init_dirs", "init_self_signed_certs", + "init_config", ] @@ -620,6 +621,28 @@ def convert_phenopacket_file(source: str, target: str): # err("failed.") +def init_config(service: str, force: bool = False): + if service == "katsu": + _init_katsu(force) + elif service == "beacon": + _init_beacon(force) + + +def _init_beacon(force: bool): + root_path = pathlib.Path.cwd() + beacon_template_path = (root_path / "etc" / "templates" / "beacon" / "beacon_config.example.json") + beacon_dest_path = (root_path / "lib" / "beacon" / "config" / "beacon_config.json") + + _file_copy(beacon_template_path, beacon_dest_path, force) + + +def _init_katsu(force: bool): + root_path = pathlib.Path.cwd() + katsu_config_template_path = (root_path / "etc" / "katsu.config.example.json") + katsu_config_dest_path = (root_path / "lib" / "katsu" / "config.json") + + _file_copy(katsu_config_template_path, katsu_config_dest_path, force) + def clean_logs(): # TODO pass From b8e8556df3f73d3a97230bab116529739aa6f67a Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 29 Apr 2024 15:00:46 -0400 Subject: [PATCH 067/130] added liniting new line --- py_bentoctl/other_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 7152dbac..0c727e20 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -643,6 +643,7 @@ def _init_katsu(force: bool): _file_copy(katsu_config_template_path, katsu_config_dest_path, force) + def clean_logs(): # TODO pass From a22d935d443c2e32ef85a99bda4b08c7ce36e89b Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Wed, 1 May 2024 00:09:24 -0400 Subject: [PATCH 068/130] moved bento_cohort.json to templates --- .../templates/beacon/beacon_cohort.example.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/beacon/config/beacon_cohort.json => etc/templates/beacon/beacon_cohort.example.json (100%) diff --git a/lib/beacon/config/beacon_cohort.json b/etc/templates/beacon/beacon_cohort.example.json similarity index 100% rename from lib/beacon/config/beacon_cohort.json rename to etc/templates/beacon/beacon_cohort.example.json From 67e57eefab122e7885625002830bd0e4fd4f9e53 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Wed, 1 May 2024 00:21:29 -0400 Subject: [PATCH 069/130] added cohort to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c1ccf64b..7d25c790 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ lib/katsu/config.json # ignored instance specific beacon config lib/beacon/config/beacon_config.json +lib/beacon/config/beacon_cohort.json # bento_web lib/web/* From e0dfeb7e89f632d8b2725676318d12c8992aaa81 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Wed, 1 May 2024 00:21:50 -0400 Subject: [PATCH 070/130] added cohort to bentoctl workflow --- py_bentoctl/other_helpers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 0c727e20..56396c16 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -630,10 +630,17 @@ def init_config(service: str, force: bool = False): def _init_beacon(force: bool): root_path = pathlib.Path.cwd() - beacon_template_path = (root_path / "etc" / "templates" / "beacon" / "beacon_config.example.json") - beacon_dest_path = (root_path / "lib" / "beacon" / "config" / "beacon_config.json") + template_dir = (root_path / "etc" / "templates" / "beacon") + dest_dir = (root_path / "lib" / "beacon" / "config") - _file_copy(beacon_template_path, beacon_dest_path, force) + config_template_path = (template_dir / "beacon_config.example.json") + config_dest_path = (dest_dir / "beacon_config.json") + + cohort_template_path = (template_dir / "beacon_cohort.example.json") + cohort_dest_path = (dest_dir / "beacon_cohort.json") + + _file_copy(config_template_path, config_dest_path, force) + _file_copy(cohort_template_path, cohort_dest_path, force) def _init_katsu(force: bool): From 87b2e642e948add7324efca6ca7a9ccb84f76ae8 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 2 May 2024 09:13:31 -0400 Subject: [PATCH 071/130] update example katsu config --- etc/katsu.config.example.json | 124 +++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 32 deletions(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index b0d61c22..65a79481 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -3,28 +3,63 @@ { "section_title": "Demographics", "charts": [ - {"field": "age", "chart_type": "bar"}, - {"field": "sex", "chart_type": "pie"}, - {"field": "date_of_consent", "chart_type": "bar"}, - {"field": "mobility", "chart_type": "bar"}, - {"field": "lab_test_result_value", "chart_type": "bar"} + { + "field": "age", + "chart_type": "bar" + }, + { + "field": "sex", + "chart_type": "pie" + }, + { + "field": "covid_severity", + "chart_type": "pie" + }, + { + "field": "date_of_consent", + "chart_type": "bar" + }, + { + "field": "mobility", + "chart_type": "bar" + }, + { + "field": "smoking_status", + "chart_type": "bar" + }, + { + "field": "lab_test_result_value", + "chart_type": "bar" + } ] }, { "section_title": "Experiments", "charts": [ - {"field": "experiment_type", "chart_type": "pie"} + { + "field": "experiment_type", + "chart_type": "pie" + } ] } ], "search": [ { "section_title": "Demographics", - "fields": ["age", "sex", "date_of_consent", "lab_test_result_value"] + "fields": [ + "age", + "sex", + "date_of_consent", + "mobility", + "covid_severity", + "lab_test_result_value" + ] }, { "section_title": "Experiments", - "fields": ["experiment_type"] + "fields": [ + "experiment_type" + ] } ], "fields": { @@ -51,16 +86,6 @@ "enum": null } }, - "experiment_type": { - "mapping": "experiment/experiment_type", - "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", - "title": "Experiment Types", - "description": "Types of experiments performed on a sample", - "datatype": "string", - "config": { - "enum": ["DNA Methylation", "mRNA-Seq", "smRNA-Seq", "RNA-Seq", "WES", "Other"] - } - }, "date_of_consent": { "mapping": "individual/extra_properties/date_of_consent", "title": "Verbal consent date", @@ -70,18 +95,6 @@ "bin_by": "month" } }, - "type_partic": { - "mapping": "individual/extra_properties/type_partic", - "title": "Participant type", - "description": "Has the patient been hospitalized or is the patient seen on as an outpatient?", - "datatype": "string", - "config": { - "enum": [ - "Hospitalized", - "Outpatient" - ] - } - }, "mobility": { "mapping": "individual/extra_properties/mobility", "title": "Functional status", @@ -97,20 +110,67 @@ ] } }, + "smoking_status": { + "mapping": "individual/extra_properties/smoking_status", + "title": "Smoking status", + "description": "Smoking status", + "datatype": "string", + "config": { + "enum": [ + "Non-smoker", + "Smoker", + "Former smoker", + "Passive smoker", + "Not specified" + ] + } + }, + "covid_severity": { + "mapping": "individual/extra_properties/covid_severity", + "title": "Covid severity", + "description": "Covid severity", + "datatype": "string", + "config": { + "enum": [ + "Uninfected", + "Mild", + "Moderate", + "Severe", + "Dead" + ] + } + }, "lab_test_result_value": { "mapping": "individual/extra_properties/lab_test_result_value", "title": "Lab Test Result", "description": "This acts as a placeholder for numeric values", "datatype": "number", "config": { - "bins": [200, 300, 500, 1000, 1500, 2000], + "bins": [ + 200, + 300, + 500, + 1000, + 1500, + 2000 + ], "minimum": 0, "units": "mg/L" } + }, + "experiment_type": { + "mapping": "experiment/experiment_type", + "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", + "title": "Experiment Types", + "description": "Types of experiments performed on a sample", + "datatype": "string", + "config": { + "enum": null + } } }, "rules": { "count_threshold": 5, "max_query_parameters": 2 } -} +} \ No newline at end of file From dae1772901481cbc9106d19c861eed80bca7341c Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Thu, 2 May 2024 11:24:33 -0400 Subject: [PATCH 072/130] add newline --- etc/katsu.config.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index 65a79481..86705e71 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -173,4 +173,4 @@ "count_threshold": 5, "max_query_parameters": 2 } -} \ No newline at end of file +} From 925565a9f96a092232c8fb25e33d258b271261e2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 3 May 2024 16:42:55 -0400 Subject: [PATCH 073/130] chore(deps): update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f4559783..61547a9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ pyhumps==3.8.0 PyYAML==6.0.1 requests==2.31.0 termcolor==2.4.0 -tqdm==4.66.2 +tqdm==4.66.4 urllib3==2.2.1 -websocket-client==1.7.0 +websocket-client==1.8.0 From 4fe0953e7f705c7836c9bf4a4de42d7283bcab02 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 6 May 2024 12:59:40 -0400 Subject: [PATCH 074/130] renamed helper functions --- py_bentoctl/other_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 56396c16..addb76e7 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -623,12 +623,12 @@ def convert_phenopacket_file(source: str, target: str): def init_config(service: str, force: bool = False): if service == "katsu": - _init_katsu(force) + _init_katsu_config(force) elif service == "beacon": - _init_beacon(force) + _init_beacon_config(force) -def _init_beacon(force: bool): +def _init_beacon_config(force: bool): root_path = pathlib.Path.cwd() template_dir = (root_path / "etc" / "templates" / "beacon") dest_dir = (root_path / "lib" / "beacon" / "config") @@ -643,7 +643,7 @@ def _init_beacon(force: bool): _file_copy(cohort_template_path, cohort_dest_path, force) -def _init_katsu(force: bool): +def _init_katsu_config(force: bool): root_path = pathlib.Path.cwd() katsu_config_template_path = (root_path / "etc" / "katsu.config.example.json") katsu_config_dest_path = (root_path / "lib" / "katsu" / "config.json") From f6ad67829bd7cb5340501ba2553d9e13f3f049b8 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 6 May 2024 13:25:58 -0400 Subject: [PATCH 075/130] make dest directory --- py_bentoctl/other_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index addb76e7..f8fcb6c0 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -632,6 +632,7 @@ def _init_beacon_config(force: bool): root_path = pathlib.Path.cwd() template_dir = (root_path / "etc" / "templates" / "beacon") dest_dir = (root_path / "lib" / "beacon" / "config") + dest_dir.mkdir(parents=True, exist_ok=True) config_template_path = (template_dir / "beacon_config.example.json") config_dest_path = (dest_dir / "beacon_config.json") From 536b30f1165bb8b0ce3a4f93dbcfda92f3c5a51e Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 6 May 2024 13:26:13 -0400 Subject: [PATCH 076/130] added changes to readme --- docs/installation.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 3783462e..998ada7c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -143,9 +143,18 @@ Bento's clinical/phenotypic metadata service: ```bash # public service configuration file. Required if BENTOV2_USE_BENTO_PUBLIC flag is set to `1` # See Katsu documentation for more information about the specifications -cp ./etc/katsu.config.example.json ./lib/katsu/config.json +./bentoctl.bash init-config katsu ``` +### Beacon configuration + +If using beacon, first copy the configuration file: + +```bash +./bentoctl.bash init-config beacon +``` + +Then update any config values as needed at `lib/beacon/config/beacon_config.json` and `lib/beacon/config/beacon_cohort.json` ### Gohan configuration From ae8eebff2e52319ad6b9084c3058368d2165305f Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 6 May 2024 13:36:11 -0400 Subject: [PATCH 077/130] readme syntax changes --- docs/installation.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 998ada7c..fba0c82a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -146,15 +146,18 @@ Bento's clinical/phenotypic metadata service: ./bentoctl.bash init-config katsu ``` + ### Beacon configuration -If using beacon, first copy the configuration file: +If using Beacon, first copy the configuration file: ```bash ./bentoctl.bash init-config beacon ``` -Then update any config values as needed at `lib/beacon/config/beacon_config.json` and `lib/beacon/config/beacon_cohort.json` +Then update any config values as needed at `lib/beacon/config/beacon_config.json` +and `lib/beacon/config/beacon_cohort.json`. + ### Gohan configuration From 2faf5479c54042d60740d51a5d94477795dfc844 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 6 May 2024 14:36:04 -0400 Subject: [PATCH 078/130] added correct permissions to the created folder --- py_bentoctl/other_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index f8fcb6c0..067dbb21 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -633,6 +633,7 @@ def _init_beacon_config(force: bool): template_dir = (root_path / "etc" / "templates" / "beacon") dest_dir = (root_path / "lib" / "beacon" / "config") dest_dir.mkdir(parents=True, exist_ok=True) + dest_dir.chmod(0o755) config_template_path = (template_dir / "beacon_config.example.json") config_dest_path = (dest_dir / "beacon_config.json") From e84daa87b5bc8fdf43f69fde963225b89221b031 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 6 May 2024 14:57:36 -0400 Subject: [PATCH 079/130] set service URL base path for authz + consistent w/ ref --- lib/authz/docker-compose.authz.yaml | 1 + lib/reference/docker-compose.reference.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/authz/docker-compose.authz.yaml b/lib/authz/docker-compose.authz.yaml index 91e3e3bd..8f4df22d 100644 --- a/lib/authz/docker-compose.authz.yaml +++ b/lib/authz/docker-compose.authz.yaml @@ -14,6 +14,7 @@ services: - INTERNAL_PORT=${BENTO_AUTHZ_INTERNAL_PORT} - OPENID_CONFIG_URL=${BENTOV2_AUTH_PUBLIC_URL}${BENTOV2_AUTH_WELLKNOWN_PATH} - CORS_ORIGINS=${BENTO_CORS_ORIGINS} + - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/authorization mem_limit: ${BENTO_AUTHZ_MEM_LIM} networks: - authz-net diff --git a/lib/reference/docker-compose.reference.yaml b/lib/reference/docker-compose.reference.yaml index 6af05f4d..9fea6f08 100644 --- a/lib/reference/docker-compose.reference.yaml +++ b/lib/reference/docker-compose.reference.yaml @@ -12,7 +12,7 @@ services: - BENTO_DEBUG=False - DATABASE_URI=postgres://${BENTO_REFERENCE_DB_USER}:${BENTO_REFERENCE_DB_PASSWORD}@${BENTO_REFERENCE_DB_CONTAINER_NAME}:5432/${BENTO_REFERENCE_DB_NAME} - INTERNAL_PORT=${BENTO_REFERENCE_INTERNAL_PORT} - - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/reference/ + - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/reference - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - BENTO_AUTHZ_SERVICE_URL healthcheck: From 25411d55dd5d1e0c5218c4d3d0b779122de0c0eb Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Wed, 8 May 2024 13:12:26 -0400 Subject: [PATCH 080/130] provided empty config folder To not deal with permission issues when creating the folder --- .gitignore | 4 ---- lib/beacon/config/.gitignore | 4 ++++ py_bentoctl/other_helpers.py | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 lib/beacon/config/.gitignore diff --git a/.gitignore b/.gitignore index 7d25c790..5e624d19 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,6 @@ lib/web/bento_web lib/katsu/config.json -# ignored instance specific beacon config -lib/beacon/config/beacon_config.json -lib/beacon/config/beacon_cohort.json - # bento_web lib/web/* !lib/web/.gitkeep diff --git a/lib/beacon/config/.gitignore b/lib/beacon/config/.gitignore new file mode 100644 index 00000000..86d0cb27 --- /dev/null +++ b/lib/beacon/config/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 067dbb21..addb76e7 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -632,8 +632,6 @@ def _init_beacon_config(force: bool): root_path = pathlib.Path.cwd() template_dir = (root_path / "etc" / "templates" / "beacon") dest_dir = (root_path / "lib" / "beacon" / "config") - dest_dir.mkdir(parents=True, exist_ok=True) - dest_dir.chmod(0o755) config_template_path = (template_dir / "beacon_config.example.json") config_dest_path = (dest_dir / "beacon_config.json") From d100e2ec4f1651e23b9c546f9e41e9fe481ec329 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Fri, 10 May 2024 16:50:20 -0400 Subject: [PATCH 081/130] add more katsu config properties --- etc/katsu.config.example.json | 155 +++++++++++++++++++++++++++++++--- 1 file changed, 143 insertions(+), 12 deletions(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index 86705e71..c199b7ca 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -1,7 +1,7 @@ { "overview": [ { - "section_title": "Demographics", + "section_title": "General", "charts": [ { "field": "age", @@ -11,14 +11,14 @@ "field": "sex", "chart_type": "pie" }, - { - "field": "covid_severity", - "chart_type": "pie" - }, { "field": "date_of_consent", "chart_type": "bar" }, + { + "field": "covid_severity", + "chart_type": "pie" + }, { "field": "mobility", "chart_type": "bar" @@ -27,9 +27,44 @@ "field": "smoking_status", "chart_type": "bar" }, + + { + "field": "phenotypic_features", + "chart_type": "pie" + }, + { + "field": "tissues", + "chart_type": "pie" + }, + { + "field": "diseases", + "chart_type": "pie" + } + ] + }, + { + "section_title": "Measurements", + "charts": [ { "field": "lab_test_result_value", "chart_type": "bar" + }, + { + "field": "bmi", + "chart_type": "bar" + } + ] + }, + { + "section_title": "Medical Actions", + "charts": [ + { + "field": "medical_procedures", + "chart_type": "pie" + }, + { + "field": "medical_treatments", + "chart_type": "pie" } ] }, @@ -39,26 +74,47 @@ { "field": "experiment_type", "chart_type": "pie" + }, + { + "field": "experiment_results_file_type", + "chart_type": "pie" } ] } ], "search": [ { - "section_title": "Demographics", + "section_title": "General", "fields": [ "age", "sex", "date_of_consent", "mobility", "covid_severity", - "lab_test_result_value" + "phenotypic_features", + "diseases", + "tissues" + ] + }, + { + "section_title": "Measurements", + "fields": [ + "lab_test_result_value", + "bmi" + ] + }, + { + "section_title": "Medical Actions", + "fields": [ + "medical_procedures", + "medical_treatments" ] }, { "section_title": "Experiments", "fields": [ - "experiment_type" + "experiment_type", + "experiment_results_file_type" ] } ], @@ -89,7 +145,7 @@ "date_of_consent": { "mapping": "individual/extra_properties/date_of_consent", "title": "Verbal consent date", - "description": "Date of initial verbal consent(participant, legal representative or tutor), yyyy-mm-dd", + "description": "Date of initial verbal consent (participant, legal representative or tutor), yyyy-mm-dd", "datatype": "date", "config": { "bin_by": "month" @@ -113,7 +169,7 @@ "smoking_status": { "mapping": "individual/extra_properties/smoking_status", "title": "Smoking status", - "description": "Smoking status", + "description": "Smoking Status", "datatype": "string", "config": { "enum": [ @@ -143,7 +199,7 @@ "lab_test_result_value": { "mapping": "individual/extra_properties/lab_test_result_value", "title": "Lab Test Result", - "description": "This acts as a placeholder for numeric values", + "description": "Numeric measures from a laboratory test", "datatype": "number", "config": { "bins": [ @@ -158,6 +214,71 @@ "units": "mg/L" } }, + "phenotypic_features": { + "mapping": "individual/phenopackets/phenotypic_features/pftype/label", + "title": "Phenotypic Features", + "description": "Individual phenotypic features, observed as either present or absent", + "datatype": "string", + "config": { + "enum": null + } + }, + "diseases": { + "mapping": "individual/phenopackets/diseases/term/label", + "title": "Diseases", + "description": "Diseases observed as either present or absent", + "datatype": "string", + "config": { + "enum": null + } + }, + "bmi": { + "mapping": "individual/phenopackets/measurements", + "group_by": "assay/id", + "group_by_value": "NCIT:C16358", + "value_mapping": "value/quantity/value", + "title": "BMI", + "description": "Body Mass Index", + "datatype": "number", + "config": { + "bins": [ + 18.5, + 30 + ], + "minimum": 0, + "units": "kg/m^2" + } + }, + "medical_procedures": { + "mapping": "individual/phenopackets/medical_actions", + "group_by": "procedure/code/label", + "title": "Medical Procedures", + "description": "A clinical procedure performed on a subject", + "datatype": "string", + "config": { + "enum": null + } + }, + "medical_treatments": { + "mapping": "individual/phenopackets/medical_actions", + "group_by": "treatment/agent/label", + "title": "Medical Treatments", + "description": "Treatment with an agent such as a drug", + "datatype": "string", + "config": { + "enum": null + } + }, + "tissues": { + "mapping": "biosample/sampled_tissue/label", + "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", + "title": "Sampled Tissues", + "description": "Tissue from which the biosample was extracted", + "datatype": "string", + "config": { + "enum": null + } + }, "experiment_type": { "mapping": "experiment/experiment_type", "mapping_for_search_filter": "individual/biosamples/experiment/experiment_type", @@ -167,10 +288,20 @@ "config": { "enum": null } + }, + "experiment_results_file_type": { + "mapping": "experiment/experiment_results/file_format", + "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", + "title": "Results File Types", + "description": "Format of experiment results files", + "datatype": "string", + "config": { + "enum": null + } } }, "rules": { - "count_threshold": 5, + "count_threshold": 3, "max_query_parameters": 2 } } From aece7ae227abd21213476193bdc2c8ef098e584f Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 13 May 2024 13:29:04 -0400 Subject: [PATCH 082/130] add study type --- etc/katsu.config.example.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index c199b7ca..6157a061 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -75,6 +75,10 @@ "field": "experiment_type", "chart_type": "pie" }, + { + "field": "experiment_study_type", + "chart_type": "pie" + }, { "field": "experiment_results_file_type", "chart_type": "pie" @@ -114,6 +118,7 @@ "section_title": "Experiments", "fields": [ "experiment_type", + "experiment_study_type", "experiment_results_file_type" ] } @@ -289,11 +294,21 @@ "enum": null } }, + "experiment_study_type": { + "mapping": "experiment/study_type", + "mapping_for_search_filter": "individual/biosamples/experiment/study_type", + "title": "Study Types", + "description": "Study type of the experiment (e.g. Genomics, Transcriptomics, etc.)", + "datatype": "string", + "config": { + "enum": null + } + }, "experiment_results_file_type": { "mapping": "experiment/experiment_results/file_format", "mapping_for_search_filter": "individual/biosamples/experiment/experiment_results/file_format", "title": "Results File Types", - "description": "Format of experiment results files", + "description": "File type of experiment results files", "datatype": "string", "config": { "enum": null From 8aefb827014130ce4c5f6956fa396b8fe4bdcb30 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 13 May 2024 14:06:52 -0400 Subject: [PATCH 083/130] add interpretations to example katsu config --- etc/katsu.config.example.json | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index 6157a061..29db3691 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -68,6 +68,19 @@ } ] }, + { + "section_title": "Interpretations", + "charts": [ + { + "field": "interpretation_status", + "chart_type": "pie" + }, + { + "field": "acmg_pathogenicity_classification", + "chart_type": "pie" + } + ] + }, { "section_title": "Experiments", "charts": [ @@ -114,6 +127,13 @@ "medical_treatments" ] }, + { + "section_title": "Interpretations", + "fields": [ + "interpretation_status", + "acmg_pathogenicity_classification" + ] + }, { "section_title": "Experiments", "fields": [ @@ -274,6 +294,24 @@ "enum": null } }, + "interpretation_status": { + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/interpretation_status", + "title": "Genomic Interpretations", + "description": "Interpretation for an individual variant or gene (CANDIDATE, CONTRIBUTORY, etc)", + "datatype": "string", + "config": { + "enum": null + } + }, + "acmg_pathogenicity_classification": { + "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", + "title": "Variant Pathogenicity Classification", + "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", + "datatype": "string", + "config": { + "enum": null + } + }, "tissues": { "mapping": "biosample/sampled_tissue/label", "mapping_for_search_filter": "individual/biosamples/sampled_tissue/label", From fb369ac854da88304734c3d33e64de7646842799 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Mon, 13 May 2024 14:09:21 -0400 Subject: [PATCH 084/130] truncate overlong chart title --- etc/katsu.config.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index 29db3691..cb42b47d 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -305,7 +305,7 @@ }, "acmg_pathogenicity_classification": { "mapping": "individual/phenopackets/interpretations/diagnosis/genomic_interpretations/variant_interpretation/acmg_pathogenicity_classification", - "title": "Variant Pathogenicity Classification", + "title": "Variant Pathogenicity", "description": "ACMG Pathogenicity category for a particular variant (BENIGN, PATHOGENIC, etc)", "datatype": "string", "config": { From 1946d0f904c09447a4078d279350097b83b32d78 Mon Sep 17 00:00:00 2001 From: Gordon Krieger Date: Tue, 14 May 2024 10:35:42 -0400 Subject: [PATCH 085/130] didn't mean to change example threshold --- etc/katsu.config.example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index cb42b47d..0a485c34 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -354,7 +354,7 @@ } }, "rules": { - "count_threshold": 3, + "count_threshold": 5, "max_query_parameters": 2 } } From 46797d4614533683fb07e0cf9aa27dba599b9bb3 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 17 May 2024 16:17:57 -0400 Subject: [PATCH 086/130] perf: tune reference db --- lib/reference/docker-compose.reference.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/reference/docker-compose.reference.yaml b/lib/reference/docker-compose.reference.yaml index 9fea6f08..5a73def0 100644 --- a/lib/reference/docker-compose.reference.yaml +++ b/lib/reference/docker-compose.reference.yaml @@ -22,6 +22,12 @@ services: reference-db: image: ${BENTO_REFERENCE_DB_IMAGE}:${BENTO_REFERENCE_DB_VERSION} + # tuning inspired by https://pgtune.leopard.in.ua/ + command: > + -c shared_buffers=256MB + -c wal_buffers=8MB + -c min_wal_size=512MB + -c max_wal_size=2GB container_name: ${BENTO_REFERENCE_DB_CONTAINER_NAME} networks: - reference-db-net From 33b36d7cefd792df198b9a9e2cae773797229a68 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 17 May 2024 16:18:06 -0400 Subject: [PATCH 087/130] chore: bump gohan to 5.0.1 --- etc/bento.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index fa7d0982..6dd12d21 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -304,14 +304,14 @@ BENTOV2_REDIS_CPUS=2 # Gohan BENTOV2_GOHAN_DEBUG=false BENTOV2_GOHAN_SERVICE_CONTACT=info@c3g.ca -BENTOV2_GOHAN_SEMVER=5.0.0 +BENTOV2_GOHAN_SEMVER=5.0.1 # -- Host BENTOV2_GOHAN_DATA_ROOT=${BENTO_FAST_DATA_DIR}/gohan # -- API BENTOV2_GOHAN_API_IMAGE=ghcr.io/bento-platform/gohan-api -BENTOV2_GOHAN_API_VERSION=5.0.0 +BENTOV2_GOHAN_API_VERSION=5.0.1 BENTOV2_GOHAN_API_VERSION_DEV=${BENTOV2_GOHAN_API_VERSION}-dev BENTOV2_GOHAN_API_CONTAINER_NAME=${BENTOV2_PREFIX}-gohan-api From 622944fb9648fb17adff90de32dccdd58a00f51f Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Thu, 23 May 2024 13:58:46 -0400 Subject: [PATCH 088/130] corrected gohan gene catalogue setup instructions --- docs/installation.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index fba0c82a..e8e699a3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -427,19 +427,22 @@ To remove the Docker containers, run the following: > (depending on data path, e.g., `./data/[auth, drs, katsu]/...` directories) -## 9. Set up Gohan's gene catalogue (*optional*; required for gene querying support) +## 9. Set Up Gohan's Gene Catalogue (Optional but Required for Gene Querying Support) -Upon initial startup of a fresh instance, it may of use, depending on the use-case, to perform the following: +To enable gene querying support, follow these steps to set up Gohan's gene catalogue: -``` -# navigate to: -https://portal.bentov2.local/api/gohan/genes/ingestion/run -# to trigger Gohan to download the default GenCode .gtk files from the internet and process them +1. **Access the Services Portal**: + - Navigate to `https://portal.bentov2.local/services`. -# - followed up by -https://portal.bentov2.local/api/gohan/genes/ingestion/requests -# to keep up with the process +2. **Initiate Gohan Request**: + - Click the `Make Request` button for Gohan. -# the end results can be found at -https://portal.bentov2.local/api/gohan/genes/overview -``` +3. **Edit and Trigger Ingestion Endpoint**: + - Modify the endpoint to `genes/ingestion/run`. + - Click `Get` to initiate Gohan's download and processing of the default GenCode `.gtk` files from the internet. + +4. **Monitor the Ingestion Process**: + - Use the endpoint `genes/ingestion/requests` to track the progress of the ingestion process. + +5. **Access the Gene Catalogue**: + - Once the ingestion process is complete, the gene catalogue will be available at `genes/overview`. From ebe21a51feeb4c994fbb849a4c9f36937f4609e9 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 27 May 2024 12:27:42 -0400 Subject: [PATCH 089/130] Made the instructions deployment independent --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index e8e699a3..1443db4e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -432,7 +432,7 @@ To remove the Docker containers, run the following: To enable gene querying support, follow these steps to set up Gohan's gene catalogue: 1. **Access the Services Portal**: - - Navigate to `https://portal.bentov2.local/services`. + - Navigate to the `Services` tab on the portal. 2. **Initiate Gohan Request**: - Click the `Make Request` button for Gohan. From de77af2b85a8028a0d5f97342c7002e7eb92c448 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 27 May 2024 14:15:32 -0400 Subject: [PATCH 090/130] set drop-box to edge + update config --- etc/bento.env | 2 +- lib/drop-box/docker-compose.drop-box.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 6dd12d21..5e96ed83 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -109,7 +109,7 @@ BENTOV2_WEB_CPUS=2 # Drop-Box BENTOV2_DROP_BOX_IMAGE=ghcr.io/bento-platform/bento_drop_box_service -BENTOV2_DROP_BOX_VERSION=1.1.5 +BENTOV2_DROP_BOX_VERSION=edge BENTOV2_DROP_BOX_VERSION_DEV=${BENTOV2_DROP_BOX_VERSION}-dev BENTOV2_DROP_BOX_CONTAINER_NAME=${BENTOV2_PREFIX}-drop-box BENTO_DROP_BOX_NETWORK=${BENTOV2_PREFIX}-drop-box-net diff --git a/lib/drop-box/docker-compose.drop-box.yaml b/lib/drop-box/docker-compose.drop-box.yaml index aee98855..cd475c82 100644 --- a/lib/drop-box/docker-compose.drop-box.yaml +++ b/lib/drop-box/docker-compose.drop-box.yaml @@ -6,7 +6,7 @@ services: - BENTO_UID - BENTO_DEBUG=False - SERVICE_DATA=${BENTO_DROP_BOX_FS_BASE_PATH} - - SERVICE_URL=${BENTOV2_PUBLIC_URL}/api/drop-box + - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/drop-box - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - BENTO_AUTHZ_SERVICE_URL - OPENID_CONFIG_URL=${BENTO_OPENID_CONFIG_URL} From dd7ff526476bdc62988368312f1e6301490af620 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 28 May 2024 15:29:41 -0400 Subject: [PATCH 091/130] set drop box to 1.1.7 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 5e96ed83..675512b7 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -109,7 +109,7 @@ BENTOV2_WEB_CPUS=2 # Drop-Box BENTOV2_DROP_BOX_IMAGE=ghcr.io/bento-platform/bento_drop_box_service -BENTOV2_DROP_BOX_VERSION=edge +BENTOV2_DROP_BOX_VERSION=1.1.7 BENTOV2_DROP_BOX_VERSION_DEV=${BENTOV2_DROP_BOX_VERSION}-dev BENTOV2_DROP_BOX_CONTAINER_NAME=${BENTOV2_PREFIX}-drop-box BENTO_DROP_BOX_NETWORK=${BENTOV2_PREFIX}-drop-box-net From 9ec58749ad8552e54936b2c22d6ea80214fd139a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 28 May 2024 15:29:50 -0400 Subject: [PATCH 092/130] set service registry to 1.4.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 675512b7..9e9789cf 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -123,7 +123,7 @@ BENTOV2_DROP_BOX_CPUS=3 # Service Registry BENTOV2_SERVICE_REGISTRY_IMAGE=ghcr.io/bento-platform/bento_service_registry -BENTOV2_SERVICE_REGISTRY_VERSION=1.3.7 +BENTOV2_SERVICE_REGISTRY_VERSION=1.4.0 BENTOV2_SERVICE_REGISTRY_VERSION_DEV=${BENTOV2_SERVICE_REGISTRY_VERSION}-dev BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME=${BENTOV2_PREFIX}-service-registry BENTO_SERVICE_REGISTRY_NETWORK=${BENTOV2_PREFIX}-service-registry-net From 7d8327289bb6f69e5f5ce925d2ae3d2d072e91db Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 31 May 2024 13:42:22 -0400 Subject: [PATCH 093/130] set reference to edge + web to pr-405 --- etc/bento.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 9e9789cf..414a4795 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=edge +BENTOV2_WEB_VERSION=pr-405 BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net @@ -179,7 +179,7 @@ BENTOV2_EVENT_RELAY_CPUS=1 # Reference # - Service BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service -BENTO_REFERENCE_VERSION=0.1.2 +BENTO_REFERENCE_VERSION=edge BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net From 03deeef0caab8f365a36ff131d29ad725510082a Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 10:14:36 -0400 Subject: [PATCH 094/130] set drs to 0.17.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9e9789cf..800606c7 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -227,7 +227,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800 # DRS BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs -BENTOV2_DRS_VERSION=edge +BENTOV2_DRS_VERSION=0.17.0 BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net From bee375bd92748c1d90230e96706c6f0ae3d013c9 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 10:17:35 -0400 Subject: [PATCH 095/130] set gateway to 0.11.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 800606c7..88000765 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=edge +BENTOV2_GATEWAY_VERSION=0.11.0 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway From f5a7bd084c25e5ced4bacdff2c5f5a372ed71324 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 12:15:57 -0400 Subject: [PATCH 096/130] set authz to 0.9.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 88000765..62f5415c 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -77,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net" # - Authz service BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service -BENTO_AUTHZ_VERSION=edge +BENTO_AUTHZ_VERSION=0.9.0 BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net From 8ce5252aa923c1984187c21061bd5794973fd01a Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 3 Jun 2024 12:22:50 -0400 Subject: [PATCH 097/130] Modified katsu config to differentiate between bar charts and histograms --- etc/katsu.config.example.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/katsu.config.example.json b/etc/katsu.config.example.json index 0a485c34..797a79f9 100644 --- a/etc/katsu.config.example.json +++ b/etc/katsu.config.example.json @@ -5,7 +5,7 @@ "charts": [ { "field": "age", - "chart_type": "bar" + "chart_type": "histogram" }, { "field": "sex", @@ -13,7 +13,7 @@ }, { "field": "date_of_consent", - "chart_type": "bar" + "chart_type": "histogram" }, { "field": "covid_severity", @@ -47,11 +47,11 @@ "charts": [ { "field": "lab_test_result_value", - "chart_type": "bar" + "chart_type": "histogram" }, { "field": "bmi", - "chart_type": "bar" + "chart_type": "histogram" } ] }, From 2fce24016744024c461664b31634859b1a3fcde5 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 14:06:00 -0400 Subject: [PATCH 098/130] bump event relay to 3.1.2 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 62f5415c..f203063c 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -165,7 +165,7 @@ BENTOV2_AGGREGATION_CPUS=2 # Event-Relay BENTOV2_EVENT_RELAY_IMAGE=ghcr.io/bento-platform/bento_event_relay -BENTOV2_EVENT_RELAY_VERSION=3.1.1 +BENTOV2_EVENT_RELAY_VERSION=3.1.2 BENTOV2_EVENT_RELAY_VERSION_DEV=${BENTOV2_EVENT_RELAY_VERSION}-dev BENTOV2_EVENT_RELAY_CONTAINER_NAME=${BENTOV2_PREFIX}-event-relay BENTO_EVENT_RELAY_NETWORK=${BENTOV2_PREFIX}-event-relay-net From 4d34210e17a4703496bddded240435dd038f55bd Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 14:11:08 -0400 Subject: [PATCH 099/130] chore: bump auth to 2024.06.03 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index f203063c..9fabe2e0 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -44,7 +44,7 @@ BENTOV2_GATEWAY_CPUS=2 # - Keycloak IdP - 'auth' BENTOV2_AUTH_IMAGE=ghcr.io/bento-platform/bento_keycloak_dist -BENTOV2_AUTH_VERSION=2024.02.16 +BENTOV2_AUTH_VERSION=2024.06.03 BENTOV2_AUTH_CONTAINER_NAME=${BENTOV2_PREFIX}-auth BENTO_AUTH_NETWORK=${BENTOV2_PREFIX}-auth-net BENTOV2_AUTH_SERVICE_HOST=0.0.0.0 From 418721d1a314a3c184703df9ebdaea1950bdbcb4 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 3 Jun 2024 15:50:20 -0400 Subject: [PATCH 100/130] Modified bento public dev docker yaml to have en /fr about html files --- docker-compose.local.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index 038caf3f..b8952ead 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -16,6 +16,8 @@ services: volumes: - ./repos/public:/bento-public - ./packs:/packs + - ${PWD}/lib/public/en_about.html:/bento-public/src/public/en_about.html + - ${PWD}/lib/public/fr_about.html:/bento-public/src/public/fr_about.html environment: - BENTO_GIT_NAME - BENTO_GIT_EMAIL From 400a08eab50c1575e0685b67b0c9eac46daf9282 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 15:58:42 -0400 Subject: [PATCH 101/130] configure reference tmp volume --- etc/bento.env | 1 + lib/reference/docker-compose.reference.yaml | 3 +++ py_bentoctl/other_helpers.py | 1 + 3 files changed, 5 insertions(+) diff --git a/etc/bento.env b/etc/bento.env index 211ae225..52dc0d55 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -187,6 +187,7 @@ BENTO_REFERENCE_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_REFERENCE_EXTERNAL_PORT=9510 BENTO_REFERENCE_DEBUGGER_INTERNAL_PORT=9511 BENTO_REFERENCE_DEBUGGER_EXTERNAL_PORT=9511 +BENTO_REFERENCE_TMP_VOL_DIR=${BENTO_FAST_DATA_DIR}/reference/tmp # - Database BENTO_REFERENCE_DB_IMAGE=postgres BENTO_REFERENCE_DB_VERSION=16 diff --git a/lib/reference/docker-compose.reference.yaml b/lib/reference/docker-compose.reference.yaml index 5a73def0..3242d78f 100644 --- a/lib/reference/docker-compose.reference.yaml +++ b/lib/reference/docker-compose.reference.yaml @@ -15,6 +15,9 @@ services: - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/reference - CORS_ORIGINS=${BENTO_CORS_ORIGINS} - BENTO_AUTHZ_SERVICE_URL + - FILE_INGEST_TMP_DIR=/reference/tmp + volumes: + - ${BENTO_REFERENCE_TMP_VOL_DIR}:/reference/tmp healthcheck: test: [ "CMD", "wget", "-q0-", "http://localhost:${BENTO_REFERENCE_INTERNAL_PORT}/service-info", "&&", "echo" ] timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT} diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index addb76e7..40dbf6ac 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -212,6 +212,7 @@ def init_dirs(): "katsu-db": "BENTOV2_KATSU_DB_PROD_VOL_DIR" if c.DEV_MODE else "BENTOV2_KATSU_DB_DEV_VOL_DIR", "notification": "BENTOV2_NOTIFICATION_VOL_DIR", "redis": "BENTOV2_REDIS_VOL_DIR", + "reference": "BENTO_REFERENCE_TMP_VOL_DIR", "reference-db": "BENTO_REFERENCE_DB_VOL_DIR", "wes": "BENTOV2_WES_VOL_DIR", "wes-tmp": "BENTOV2_WES_VOL_TMP_DIR", From efbfc78d490dc15ef51fda1dc8d67f629e5a93cb Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 15:59:00 -0400 Subject: [PATCH 102/130] docs: update v16 migration guide --- docs/migrating_to_16.md | 59 +++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index 76fb1a82..bb18a9cb 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -11,16 +11,51 @@ Key points: * If needed, the default service specific volume directories defined in [bento.env](../etc/bento.env) can be overriden in your `local.env`. * The WES schema has been updated, so the run database must be cleared. +* The reference service has been updated, so a SQL migration must be run. -## 1. Pre-update data config +## 1. Run reference service pre-migration step + +```bash +./bentoctl.bash shell reference-db +PGPASSWORD="${POSTGRES_PASSWORD}" psql --user "${POSTGRES_USER}" reference +``` + +Then, begin a transaction: + +```sql +BEGIN TRANSACTION; +``` + +Then, paste the contents of +[migrate_v0_2.sql](https://github.com/bento-platform/bento_reference_service/blob/main/bento_reference_service/sql/migrate_v0_2.sql) +from the reference service repository into the SQL command line. + +Next, commit the results of the transaction: + +```sql +COMMIT; +``` + +Finally, exit the shell (via successive `^D`). + + +## 2. Stop Bento + +```bash +./bentoctl.bash stop +``` + + +## 3. Pre-update data config Before updating, perform the following steps: * Shut down Bento with `./bentoctl.bash stop` * In `local.env`, replace `BENTOV2_ROOT_DATA_DIR` with `BENTO_FAST_DATA_DIR` and `BENTO_SLOW_DATA_DIR` -To minimize side effects in local environments, we recommend that you use the same directory as before for both new variables. +To minimize side effects in local environments, we recommend that you use the same directory as before for both new +variables. ```bash # local.env @@ -33,28 +68,30 @@ BENTO_FAST_DATA_DIR=some_dir BENTO_SLOW_DATA_DIR=some_dir ``` -*IMPORTANT:* If you decide to split the data in distinct directories, make sure to migrate the data accordingly. +*IMPORTANT:* If you decide to split the data in distinct directories, **make sure to migrate the data accordingly**. -## 2. Update images +## 4. Update images ```bash ./bentoctl.bash pull ``` -## 3. Create new Docker volume directories - -This step is optional if you DIDN'T split the data directories in step 1. +## 5. Create new Docker volume directories ```bash -./bentoctl.bash init-dirs # new volume directory for DRS temporary files - -# migrate the old data +./bentoctl.bash init-dirs # new volume directory for DRS and Reference temporary files ``` -## 4. Restart Bento +## 6. Migrate data from old `BENTOV2_ROOT_DATA_DIR` location to `BENTO_FAST_DATA_DIR`/`BENTO_SLOW_DATA_DIR` + +This is only needed if the data was split into distinct directories in part 1. All service data should be copied into +the corresponding locations as specified in `etc/bento.env`. + + +## 7. Restart Bento ```bash ./bentoctl.bash start From 9f6a54d28d37f3315e8a11c9b0b56b213ed3d79e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 16:17:59 -0400 Subject: [PATCH 103/130] set reference to 0.2.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 52dc0d55..555c9ca3 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -179,7 +179,7 @@ BENTOV2_EVENT_RELAY_CPUS=1 # Reference # - Service BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service -BENTO_REFERENCE_VERSION=edge +BENTO_REFERENCE_VERSION=0.2.0 BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net From 4a5ae600cc54f2a068d3e0a2345cd7b239d3a0de Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 17:05:12 -0400 Subject: [PATCH 104/130] set aggregation to 0.19.4 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9fabe2e0..bc1ab4cc 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -150,7 +150,7 @@ BENTOV2_NOTIFICATION_CPUS=2 # Aggregation BENTOV2_AGGREGATION_IMAGE=ghcr.io/bento-platform/bento_aggregation_service -BENTOV2_AGGREGATION_VERSION=0.19.3 +BENTOV2_AGGREGATION_VERSION=0.19.4 BENTOV2_AGGREGATION_VERSION_DEV=${BENTOV2_AGGREGATION_VERSION}-dev BENTOV2_AGGREGATION_CONTAINER_NAME=${BENTOV2_PREFIX}-aggregation BENTO_AGGREGATION_NETWORK=${BENTOV2_PREFIX}-aggregation-net From 232fa74d9ce318f23f7249a6262c03e8c111a110 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 3 Jun 2024 17:06:25 -0400 Subject: [PATCH 105/130] set wes to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index bc1ab4cc..b15606d7 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -200,7 +200,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=0.13.0 +BENTOV2_WES_VERSION=edge BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net From f5ade1e541689d44bf90c43761067f5282129997 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 5 Jun 2024 16:35:34 -0400 Subject: [PATCH 106/130] docs: reformat 16 migration guide --- docs/migrating_to_16.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index bb18a9cb..4f633a38 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -11,14 +11,15 @@ Key points: * If needed, the default service specific volume directories defined in [bento.env](../etc/bento.env) can be overriden in your `local.env`. * The WES schema has been updated, so the run database must be cleared. -* The reference service has been updated, so a SQL migration must be run. +* The reference service has been updated, so an SQL migration must be run. ## 1. Run reference service pre-migration step ```bash ./bentoctl.bash shell reference-db -PGPASSWORD="${POSTGRES_PASSWORD}" psql --user "${POSTGRES_USER}" reference +PGPASSWORD="${POSTGRES_PASSWORD}" +psql --user "${POSTGRES_USER}" reference ``` Then, begin a transaction: From 2cfc7d7a1686804252cedced21a4cb294bc5dd2b Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 10 Jun 2024 16:16:35 -0400 Subject: [PATCH 107/130] set web to edge --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 4ca67d76..b0f256bd 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=pr-405 +BENTOV2_WEB_VERSION=edge BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net From f5a43d7924b86620e9c9b0c76c735870b8151135 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Wed, 12 Jun 2024 13:03:38 -0400 Subject: [PATCH 108/130] added all public content to volume --- docker-compose.local.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index b8952ead..598f05da 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -16,8 +16,7 @@ services: volumes: - ./repos/public:/bento-public - ./packs:/packs - - ${PWD}/lib/public/en_about.html:/bento-public/src/public/en_about.html - - ${PWD}/lib/public/fr_about.html:/bento-public/src/public/fr_about.html + - ${PWD}/lib/public:/bento-public/src/public environment: - BENTO_GIT_NAME - BENTO_GIT_EMAIL From ff953bda0c4d60be449f8a9779bfcdec5e67a9d3 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 17 Jun 2024 10:58:48 -0400 Subject: [PATCH 109/130] modified local docker compose for public to only copy instance specific files --- docker-compose.local.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index 598f05da..54f2ae8b 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -16,7 +16,11 @@ services: volumes: - ./repos/public:/bento-public - ./packs:/packs - - ${PWD}/lib/public:/bento-public/src/public + - ${PWD}/lib/public/en_about.html:/bento-public/src/public/en_about.html + - ${PWD}/lib/public/fr_about.html:/bento-public/src/public/fr_about.html + - ${PWD}/lib/public/branding.png:/bento-public/src/public/branding.png + - ${PWD}/lib/public/translations/en.json:/bento-public/src/public/locales/en/translation.json + - ${PWD}/lib/public/translations/fr.json:/bento-public/src/public/locales/fr/translation.json environment: - BENTO_GIT_NAME - BENTO_GIT_EMAIL From 0a925cf1f0ce45bf073f523f06c2a102c8c4c511 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 17 Jun 2024 11:44:59 -0400 Subject: [PATCH 110/130] corrected path --- docker-compose.local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index 54f2ae8b..c78bda35 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -18,7 +18,7 @@ services: - ./packs:/packs - ${PWD}/lib/public/en_about.html:/bento-public/src/public/en_about.html - ${PWD}/lib/public/fr_about.html:/bento-public/src/public/fr_about.html - - ${PWD}/lib/public/branding.png:/bento-public/src/public/branding.png + - ${PWD}/lib/public/branding.png:/bento-public/src/public/assets/branding.png - ${PWD}/lib/public/translations/en.json:/bento-public/src/public/locales/en/translation.json - ${PWD}/lib/public/translations/fr.json:/bento-public/src/public/locales/fr/translation.json environment: From 839dd0a97f64700a7814fea8baa0064426317314 Mon Sep 17 00:00:00 2001 From: Sanjeev Lakhwani Date: Mon, 17 Jun 2024 12:04:05 -0400 Subject: [PATCH 111/130] corrected typo --- docker-compose.local.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index c78bda35..4962a249 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -19,8 +19,8 @@ services: - ${PWD}/lib/public/en_about.html:/bento-public/src/public/en_about.html - ${PWD}/lib/public/fr_about.html:/bento-public/src/public/fr_about.html - ${PWD}/lib/public/branding.png:/bento-public/src/public/assets/branding.png - - ${PWD}/lib/public/translations/en.json:/bento-public/src/public/locales/en/translation.json - - ${PWD}/lib/public/translations/fr.json:/bento-public/src/public/locales/fr/translation.json + - ${PWD}/lib/public/translations/en.json:/bento-public/src/public/locales/en/translation_en.json + - ${PWD}/lib/public/translations/fr.json:/bento-public/src/public/locales/fr/translation_fr.json environment: - BENTO_GIT_NAME - BENTO_GIT_EMAIL From 79384cdd391058903a98ad86fa7dc2d23ccec07f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 17 Jun 2024 13:28:38 -0400 Subject: [PATCH 112/130] set wes to 0.14.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index b0f256bd..386d7222 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -201,7 +201,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=edge +BENTOV2_WES_VERSION=0.14.0 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net From 3e099bbebcbd7252641327d10a914dcf735de5d8 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 17 Jun 2024 16:41:01 -0400 Subject: [PATCH 113/130] docs: update authz grants instructions for reference genome --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 1443db4e..083426a9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -331,7 +331,7 @@ bento_authz create grant \ bento_authz create grant \ '{"iss": "ISSUER_HERE", "client": "wes"}' \ '{"everything": true}' \ - 'query:data' 'ingest:data' + 'query:data' 'ingest:data' 'ingest:reference_material' 'delete:reference_material' ``` ### c. *Optional step:* Assign portal access to all users in the instance realm From 7e6eb63d48600fff77ba90c8d4d725275fb25556 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 17 Jun 2024 16:54:03 -0400 Subject: [PATCH 114/130] set beacon to 0.15.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 386d7222..7a3e9e29 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -392,7 +392,7 @@ BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=edge +BENTO_BEACON_VERSION=0.15.0 BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 From 2b4bd4c84b680443ded23f898b2824efa1c047af Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 17 Jun 2024 16:57:33 -0400 Subject: [PATCH 115/130] update comment --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 083426a9..b4f680f0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -327,7 +327,7 @@ bento_authz create grant \ '{"everything": true}' \ 'view:private_portal' -# This grant gives permission to access and ingest data into all projects +# This grant gives permission to access and ingest data into all projects and the reference genome service bento_authz create grant \ '{"iss": "ISSUER_HERE", "client": "wes"}' \ '{"everything": true}' \ From 988f05d0508d90f5ec12b23722c187e9d9b661e4 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 18 Jun 2024 12:47:39 -0400 Subject: [PATCH 116/130] chore: bump WES to 0.14.1 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 7a3e9e29..37c12502 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -201,7 +201,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=0.14.0 +BENTOV2_WES_VERSION=0.14.1 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net From 9a04783719c2aa9a2d0008dd37f5e7d4230bf45d Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 9 Jul 2024 12:01:12 -0400 Subject: [PATCH 117/130] chore: bumb docker-py version to 7.1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 152e8e9f..650ed2c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cffi==1.16.0 charset-normalizer==3.3.2 cryptography==42.0.5 debugpy==1.8.1 -docker==7.0.0 +docker==7.1.0 flake8==7.0.0 idna==3.7 mccabe==0.7.0 From 65bfb10150ed94d3959f9b52a01a9f799c3e33fa Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 9 Jul 2024 12:27:16 -0400 Subject: [PATCH 118/130] chore: update other bentoctl dependencies --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 650ed2c8..50b093b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ certifi==2024.7.4 cffi==1.16.0 charset-normalizer==3.3.2 -cryptography==42.0.5 -debugpy==1.8.1 +cryptography==42.0.8 +debugpy==1.8.2 docker==7.1.0 -flake8==7.0.0 +flake8==7.1.0 idna==3.7 mccabe==0.7.0 -packaging==24.0 -pycodestyle==2.11.1 +packaging==24.1 +pycodestyle==2.12.0 pycparser==2.22 pyflakes==3.2.0 pyhumps==3.8.0 PyYAML==6.0.1 -requests==2.32.2 +requests==2.32.3 termcolor==2.4.0 tqdm==4.66.4 urllib3==2.2.2 From a095e3f4dec6704210f23906e07d78bb96b707a2 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 15 Jul 2024 17:07:16 -0400 Subject: [PATCH 119/130] set web to 5.0.0, public to 0.19.0; other service patch versions --- etc/bento.env | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/etc/bento.env b/etc/bento.env index 37c12502..77163b32 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -77,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net" # - Authz service BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service -BENTO_AUTHZ_VERSION=0.9.0 +BENTO_AUTHZ_VERSION=0.9.1 BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=edge +BENTOV2_WEB_VERSION=5.0.0 BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net @@ -109,7 +109,7 @@ BENTOV2_WEB_CPUS=2 # Drop-Box BENTOV2_DROP_BOX_IMAGE=ghcr.io/bento-platform/bento_drop_box_service -BENTOV2_DROP_BOX_VERSION=1.1.7 +BENTOV2_DROP_BOX_VERSION=1.1.8 BENTOV2_DROP_BOX_VERSION_DEV=${BENTOV2_DROP_BOX_VERSION}-dev BENTOV2_DROP_BOX_CONTAINER_NAME=${BENTOV2_PREFIX}-drop-box BENTO_DROP_BOX_NETWORK=${BENTOV2_PREFIX}-drop-box-net @@ -123,7 +123,7 @@ BENTOV2_DROP_BOX_CPUS=3 # Service Registry BENTOV2_SERVICE_REGISTRY_IMAGE=ghcr.io/bento-platform/bento_service_registry -BENTOV2_SERVICE_REGISTRY_VERSION=1.4.0 +BENTOV2_SERVICE_REGISTRY_VERSION=1.4.1 BENTOV2_SERVICE_REGISTRY_VERSION_DEV=${BENTOV2_SERVICE_REGISTRY_VERSION}-dev BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME=${BENTOV2_PREFIX}-service-registry BENTO_SERVICE_REGISTRY_NETWORK=${BENTOV2_PREFIX}-service-registry-net @@ -135,7 +135,7 @@ BENTOV2_SERVICE_REGISTRY_CPUS=1 # Notification BENTOV2_NOTIFICATION_IMAGE=ghcr.io/bento-platform/bento_notification_service -BENTOV2_NOTIFICATION_VERSION=3.1.2 +BENTOV2_NOTIFICATION_VERSION=3.1.4 BENTOV2_NOTIFICATION_VERSION_DEV=${BENTOV2_NOTIFICATION_VERSION}-dev BENTOV2_NOTIFICATION_CONTAINER_NAME=${BENTOV2_PREFIX}-notification BENTO_NOTIFICATION_NETWORK=${BENTOV2_PREFIX}-notification-net @@ -150,7 +150,7 @@ BENTOV2_NOTIFICATION_CPUS=2 # Aggregation BENTOV2_AGGREGATION_IMAGE=ghcr.io/bento-platform/bento_aggregation_service -BENTOV2_AGGREGATION_VERSION=0.19.4 +BENTOV2_AGGREGATION_VERSION=0.19.5 BENTOV2_AGGREGATION_VERSION_DEV=${BENTOV2_AGGREGATION_VERSION}-dev BENTOV2_AGGREGATION_CONTAINER_NAME=${BENTOV2_PREFIX}-aggregation BENTO_AGGREGATION_NETWORK=${BENTOV2_PREFIX}-aggregation-net @@ -165,7 +165,7 @@ BENTOV2_AGGREGATION_CPUS=2 # Event-Relay BENTOV2_EVENT_RELAY_IMAGE=ghcr.io/bento-platform/bento_event_relay -BENTOV2_EVENT_RELAY_VERSION=3.1.2 +BENTOV2_EVENT_RELAY_VERSION=3.1.3 BENTOV2_EVENT_RELAY_VERSION_DEV=${BENTOV2_EVENT_RELAY_VERSION}-dev BENTOV2_EVENT_RELAY_CONTAINER_NAME=${BENTOV2_PREFIX}-event-relay BENTO_EVENT_RELAY_NETWORK=${BENTOV2_PREFIX}-event-relay-net @@ -179,7 +179,7 @@ BENTOV2_EVENT_RELAY_CPUS=1 # Reference # - Service BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service -BENTO_REFERENCE_VERSION=0.2.0 +BENTO_REFERENCE_VERSION=0.2.1 BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net @@ -201,7 +201,7 @@ BENTO_REFERENCE_DB_USER="reference_user" # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes -BENTOV2_WES_VERSION=0.14.1 +BENTOV2_WES_VERSION=0.14.2 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net @@ -228,7 +228,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800 # DRS BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs -BENTOV2_DRS_VERSION=0.17.0 +BENTOV2_DRS_VERSION=0.17.1 BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net @@ -341,7 +341,7 @@ BENTOV2_GOHAN_ES_USERNAME=elastic # BENTOV2_GOHAN_ES_PASSWORD comes from default_config BENTOV2_GOHAN_ES_IMAGE=ghcr.io/bento-platform/gohan-elasticsearch -BENTOV2_GOHAN_ES_VERSION=5.0.0 +BENTOV2_GOHAN_ES_VERSION=5.0.1 BENTOV2_GOHAN_ES_CONTAINER_NAME=${BENTOV2_PREFIX}-gohan-elasticsearch BENTO_GOHAN_ES_NETWORK=${BENTOV2_PREFIX}-gohan-elasticsearch-net @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=edge +BENTO_PUBLIC_VERSION=0.19.0 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net @@ -392,7 +392,7 @@ BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=0.15.0 +BENTO_BEACON_VERSION=0.15.1 BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 @@ -436,7 +436,7 @@ BENTO_CBIOPORTAL_SESSION_SERVER_PORT=5000 # Need mongo 6.0+ to support arm64 + x86_64 (both needed for development) # Technically not in line with the cBioPortal requirements... we'll see if it causes issues BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE=mongo -BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.7 +BENTO_CBIOPORTAL_SESSION_DATABASE_IMAGE_VERSION=6.0.15 BENTO_CBIOPORTAL_SESSION_DATABASE_CONTAINER_NAME=${BENTOV2_PREFIX}-cbioportal-session-db BENTO_CBIOPORTAL_SESSION_DATABASE_DATA_DIR=${BENTO_FAST_DATA_DIR}/cbioportal/session_db # Uses BENTO_CBIOPORTAL_SESSION_NETWORK as the network From d2081512216536d8fbabb2cb2b85ff03b1c03ff8 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 16 Jul 2024 14:28:07 -0400 Subject: [PATCH 120/130] chore: bump gateway to v0.12.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 37c12502..cc5dff58 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs # Gateway BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway -BENTOV2_GATEWAY_VERSION=0.11.0 +BENTOV2_GATEWAY_VERSION=0.12.0 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway From a2b795776beac7fa147cb81df8bc65c772cb8fec Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 16 Jul 2024 15:15:30 -0400 Subject: [PATCH 121/130] chore: bump beacon version to v0.15.2 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index cc5dff58..5f689f37 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -392,7 +392,7 @@ BENTO_PUBLIC_PORTAL_URL=${BENTOV2_PORTAL_PUBLIC_URL} BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon -BENTO_BEACON_VERSION=0.15.0 +BENTO_BEACON_VERSION=0.15.2 BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} BENTO_BEACON_EXTERNAL_PORT=5000 From cab5335fb42eba4272a2af339e2cf10202bef35f Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 16 Jul 2024 15:28:47 -0400 Subject: [PATCH 122/130] chore: bump bento public version to 0.19.1 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 5f689f37..9d877f57 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -371,7 +371,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME} # Bento-Public BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public -BENTO_PUBLIC_VERSION=edge +BENTO_PUBLIC_VERSION=0.19.1 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net From ffe9ad4d417db13e419296e0466da36cd69ce220 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 16 Jul 2024 15:36:54 -0400 Subject: [PATCH 123/130] chore: bump katsu to 8.0.0 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 9d877f57..4bc82ddf 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -269,7 +269,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=edge +BENTOV2_KATSU_VERSION=8.0.0 BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net From 4b0193d54a4ca9e99aeed11946357964f3dbc131 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Wed, 17 Jul 2024 18:25:05 -0400 Subject: [PATCH 124/130] chore: bump katsu version to 8.0.1 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 0860a2cc..8c43e0f4 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -269,7 +269,7 @@ BENTOV2_KATSU_DB_CPUS=4 # Katsu BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu -BENTOV2_KATSU_VERSION=8.0.0 +BENTOV2_KATSU_VERSION=8.0.1 BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net From 86e07c208487da4abe94c7f15c49426ee84286d1 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 18 Jul 2024 13:42:50 -0400 Subject: [PATCH 125/130] docs: new ingestion instructions for reference material --- docs/installation.md | 19 +++++++++++++++++-- docs/migrating_to_16.md | 8 ++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index b4f680f0..d70f84ce 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -427,9 +427,24 @@ To remove the Docker containers, run the following: > (depending on data path, e.g., `./data/[auth, drs, katsu]/...` directories) -## 9. Set Up Gohan's Gene Catalogue (Optional but Required for Gene Querying Support) +## 9. Ingest reference material -To enable gene querying support, follow these steps to set up Gohan's gene catalogue: +To support genomic data and gene querying, a reference genome (FASTA) and GFF3 gene catalog must be ingested. +Follow these steps: + +1. Download your reference genome and GFF3 gene catalog files from RefSeq or a similar database. +2. Upload these files to your Bento instance's drop box. +3. Have a Phenopackets JSON-style ontology term (from the + [`NCBITaxon` ontology](https://github.com/obophenotype/ncbitaxon)) ready, e.g.: + ```json + {"id": "NCBITaxon:9606", "label": "Homo sapiens"} + ``` +4. Navigate to `/data/manager/ingestion` in the Bento private portal and ingest the reference FASTA and GFF3 into the + reference service. This will take some time. + +### Set Up Gohan's Gene Catalogue (*Legacy instructions*) + +To enable gene querying support in versions of Bento before v16, follow these steps to set up Gohan's gene catalogue: 1. **Access the Services Portal**: - Navigate to the `Services` tab on the portal. diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index 4f633a38..226d1019 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -12,6 +12,9 @@ Key points: `local.env`. * The WES schema has been updated, so the run database must be cleared. * The reference service has been updated, so an SQL migration must be run. +* The reference service now supports GFF3 (gene feature annotation) files, and the private portal has moved to using + this service to provide gene information for searching and track visualization. **All annotations required should be + ingested/attached to the reference genome.** ## 1. Run reference service pre-migration step @@ -97,3 +100,8 @@ the corresponding locations as specified in `etc/bento.env`. ```bash ./bentoctl.bash start ``` + + +## 8. If needed, ingest gene features + +TODO From af55508d4c790e2efde0b2c821d59f87701435fc Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 19 Jul 2024 14:19:15 -0400 Subject: [PATCH 126/130] bump web to 5.0.1 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 8c43e0f4..1328d2d5 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -97,7 +97,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G # Web BENTO_WEB_CUSTOM_HEADER= BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web -BENTOV2_WEB_VERSION=5.0.0 +BENTOV2_WEB_VERSION=5.0.1 BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net From 1b690c9f898764cc71fc7621dc1fd39e55c04404 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 19 Jul 2024 14:46:47 -0400 Subject: [PATCH 127/130] bump authz to 0.9.2 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 1328d2d5..479f6914 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -77,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net" # - Authz service BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service -BENTO_AUTHZ_VERSION=0.9.1 +BENTO_AUTHZ_VERSION=0.9.2 BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net From ae8714197b0fcd43fd8d09a148df6dcf92a22067 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Sun, 21 Jul 2024 16:02:30 -0400 Subject: [PATCH 128/130] bump reference to 0.2.2 --- etc/bento.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bento.env b/etc/bento.env index 479f6914..7ec69b2f 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -179,7 +179,7 @@ BENTOV2_EVENT_RELAY_CPUS=1 # Reference # - Service BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service -BENTO_REFERENCE_VERSION=0.2.1 +BENTO_REFERENCE_VERSION=0.2.2 BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net From 87326c009173c8c8cbecac073eac2196ad2493d8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 22 Jul 2024 09:39:38 -0400 Subject: [PATCH 129/130] docs: add note on ingesting genes for v16 --- docs/migrating_to_16.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index 226d1019..63c11039 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -104,4 +104,7 @@ the corresponding locations as specified in `etc/bento.env`. ## 8. If needed, ingest gene features -TODO +Use the ingestion workflows now provided by the reference service to either ingest a reference genome FASTA with a +corresponding GFF3 file, or ingest a GFF3 file to attach to an existing reference genome. + +**Do this if the instance was previously using the Gohan gene catalog.** From c1e8dd8f62f747a762cca39246b11e2c8c73b6e9 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 23 Jul 2024 16:52:50 -0400 Subject: [PATCH 130/130] docs: v16 migration docker notes --- docs/migrating_to_16.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/migrating_to_16.md b/docs/migrating_to_16.md index 63c11039..2a1d8768 100644 --- a/docs/migrating_to_16.md +++ b/docs/migrating_to_16.md @@ -15,6 +15,10 @@ Key points: * The reference service now supports GFF3 (gene feature annotation) files, and the private portal has moved to using this service to provide gene information for searching and track visualization. **All annotations required should be ingested/attached to the reference genome.** +* Docker requirements for deployment have changed. + * Docker >= 24.0.4 + * Docker Compose >= 2.20.0 (plugin) + * Update the versions on the host machine accordingly ## 1. Run reference service pre-migration step