diff --git a/.gitignore b/.gitignore index 5e624d19..a0759800 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ lib/public/* lib/gohan/es_jvm_options/* !lib/gohan/es_jvm_options/.gitkeep +# TDS +lib/tds/plugins/* + # temp test stuff etc/tests/*/__pycache__ etc/tests/*/*/__pycache__ diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 243e6bb1..93e79b5b 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -85,6 +85,11 @@ services: - ${BENTOV2_DOMAIN} - ${BENTOV2_PORTAL_DOMAIN} - ${BENTOV2_AUTH_DOMAIN} + tds-net: + aliases: + - ${BENTOV2_DOMAIN} + - ${BENTOV2_PORTAL_DOMAIN} + - ${BENTOV2_AUTH_DOMAIN} web-net: aliases: - ${BENTOV2_DOMAIN} @@ -224,6 +229,19 @@ services: ports: - "${BENTO_REFERENCE_DB_EXTERNAL_PORT}:5432" + tds: + environment: + - BENTO_DEBUG=True + - BENTO_VALIDATE_SSL=False + - LOG_LEVEL=debug + ports: + - "${BENTO_TDS_EXTERNAL_PORT}:${BENTO_TDS_INTERNAL_PORT}" + - "${BENTO_TDS_DEBUGGER_EXTERNAL_PORT}:${BENTO_TDS_DEBUGGER_INTERNAL_PORT}" + + tds-db: + ports: + - "${BENTO_TDS_DB_EXTERNAL_PORT}:5432" + gohan-elasticsearch: # In dev / on Mac, sometimes this will trip up and decide there isn't enough disk space even when there is. # See https://stackoverflow.com/questions/48119189/elasticsearch-6-index-change-to-read-only-after-few-second diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml index 5dbd201f..81500a80 100644 --- a/docker-compose.local.yaml +++ b/docker-compose.local.yaml @@ -156,3 +156,13 @@ services: volumes: - ./repos/gohan-api:/gohan-api - ./repos/gohan-api/src/api/workflows:/app/workflows + + tds: + image: ${BENTO_TDS_IMAGE}:${BENTO_TDS_VERSION_DEV} + environment: + - BENTO_GIT_NAME + - BENTO_GIT_EMAIL + - BENTO_GIT_REPOSITORY_DIR=/tds + volumes: + - ./repos/tds:/tds + diff --git a/docker-compose.yaml b/docker-compose.yaml index f91aaa52..a6921c90 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,6 +19,7 @@ include: - lib/redis/docker-compose.redis.yaml - lib/reference/docker-compose.reference.yaml - lib/service-registry/docker-compose.service-registry.yaml + - lib/tds/docker-compose.tds.yaml - lib/web/docker-compose.web.yaml - lib/wes/docker-compose.wes.yaml project_directory: . # Paths in the lib/* compose files must be relative to the Bento base directory diff --git a/docs/development.md b/docs/development.md index c357bb1c..93226119 100644 --- a/docs/development.md +++ b/docs/development.md @@ -140,6 +140,8 @@ The following is a list of all host port allocations for Bento services in devel | Reference | 9510 | 9501 | | Reference DB | 9512 | `N/A` | | Service Registry | 5010 | Unimplemented | +| TDS | 9520 | 9520 | +| TDS DB | 9522 | `N/A` | | WES | 9250 | 5680 | diff --git a/etc/bento.env b/etc/bento.env index a07ef355..0ff03e11 100644 --- a/etc/bento.env +++ b/etc/bento.env @@ -204,6 +204,29 @@ BENTO_REFERENCE_DB_NAME="reference" BENTO_REFERENCE_DB_USER="reference_user" # BENTO_REFERENCE_DB_PASSWORD is set in local.env +# Transcriptomics Data Service +# Service: +BENTO_TDS_IMAGE=ghcr.io/bento-platform/transcriptomics_data_service +BENTO_TDS_VERSION=pr-8 +BENTO_TDS_VERSION_DEV=${BENTO_TDS_VERSION}-dev +BENTO_TDS_CONTAINER_NAME=${BENTOV2_PREFIX}-tds +BENTO_TDS_NETWORK=${BENTOV2_PREFIX}-tds-net +BENTO_TDS_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT} +BENTO_TDS_EXTERNAL_PORT=9520 +BENTO_TDS_DEBUGGER_INTERNAL_PORT=9511 +BENTO_TDS_DEBUGGER_EXTERNAL_PORT=9511 +BENTO_TDS_PLUGINS_DIR=${PWD}/lib/tds/plugins +# Database +BENTO_TDS_DB_IMAGE=postgres +BENTO_TDS_DB_VERSION=16 +BENTO_TDS_DB_CONTAINER_NAME=${BENTOV2_PREFIX}-tds-db +BENTO_TDS_DB_NETWORK=${BENTOV2_PREFIX}-tds-db-net +BENTO_TDS_DB_VOL_DIR=${BENTO_FAST_DATA_DIR}/tds/data +BENTO_TDS_DB_EXTERNAL_PORT=9522 +BENTO_TDS_DB_NAME="tds" +BENTO_TDS_DB_USER="tds_user" +# BENTO_TDS_DB_PASSWORD is set in local.env + # WES BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes BENTOV2_WES_VERSION=0.14.5 diff --git a/etc/bento_services.json b/etc/bento_services.json index 6980180e..1885d9d9 100644 --- a/etc/bento_services.json +++ b/etc/bento_services.json @@ -71,5 +71,10 @@ }, "gateway": { "repository": "git@github.com:bento-platform/bento_gateway.git" + }, + "tds": { + "service_kind": "tds", + "url_template": "{BENTO_PORTAL_PUBLIC_URL}/api/{service_kind}", + "repository": "git@github.com:bento-platform/transcriptomics_data_service.git" } } diff --git a/lib/gateway/docker-compose.gateway.yaml b/lib/gateway/docker-compose.gateway.yaml index c0a9ef65..e7019d8a 100644 --- a/lib/gateway/docker-compose.gateway.yaml +++ b/lib/gateway/docker-compose.gateway.yaml @@ -78,6 +78,8 @@ services: - BENTO_CBIOPORTAL_CONTAINER_NAME - BENTO_CBIOPORTAL_INTERNAL_PORT - BENTO_GRAFANA_CONTAINER_NAME + - BENTO_TDS_CONTAINER_NAME + - BENTO_TDS_INTERNAL_PORT networks: - aggregation-net - auth-net @@ -94,6 +96,7 @@ services: - public-net - reference-net - service-registry-net + - tds-net - web-net - wes-net ports: @@ -186,6 +189,9 @@ networks: service-registry-net: external: true name: ${BENTO_SERVICE_REGISTRY_NETWORK} + tds-net: + external: true + name: ${BENTO_TDS_NETWORK} web-net: external: true name: ${BENTO_WEB_NETWORK} diff --git a/lib/gateway/services/tds.conf.tpl b/lib/gateway/services/tds.conf.tpl new file mode 100644 index 00000000..a8ba103a --- /dev/null +++ b/lib/gateway/services/tds.conf.tpl @@ -0,0 +1,15 @@ +location /api/tds { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/tds/; } +location /api/tds/ { + # Reverse proxy settings + include /gateway/conf/proxy.conf; + include /gateway/conf/proxy_extra.conf; + + # Forward request to TDS + rewrite ^ $request_uri; + rewrite ^/api/tds/(.*) /$1 break; + return 400; + proxy_pass http://${BENTO_TDS_CONTAINER_NAME}:${BENTO_TDS_INTERNAL_PORT}$uri; + + # Errors + error_log /var/log/bento_tds_errors.log; +} diff --git a/lib/tds/docker-compose.tds.yaml b/lib/tds/docker-compose.tds.yaml new file mode 100644 index 00000000..e5b5664d --- /dev/null +++ b/lib/tds/docker-compose.tds.yaml @@ -0,0 +1,59 @@ +services: + tds: + image: ${BENTO_TDS_IMAGE}:${BENTO_TDS_VERSION} + container_name: ${BENTO_TDS_CONTAINER_NAME} + networks: + - tds-net + - tds-db-net + expose: + - ${BENTO_TDS_INTERNAL_PORT} + depends_on: + tds-db: + condition: service_healthy + environment: + - BENTO_UID + - BENTO_DEBUG=False + - DATABASE_URI=postgres://${BENTO_TDS_DB_USER}:${BENTO_TDS_DB_PASSWORD}@${BENTO_TDS_DB_CONTAINER_NAME}:5432/${BENTO_TDS_DB_NAME} + - INTERNAL_PORT=${BENTO_TDS_INTERNAL_PORT} + - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/tds + - CORS_ORIGINS=${BENTO_CORS_ORIGINS} + - BENTO_AUTHZ_SERVICE_URL + - BENTO_AUTHZ_ENABLED=True + volumes: + - ${BENTO_TDS_PLUGINS_DIR}:/tds/lib + healthcheck: + test: ["CMD", "curl", "http://localhost:${BENTO_TDS_INTERNAL_PORT}/service-info"] + timeout: ${BENTO_HEALTHCHECK_TIMEOUT} + interval: ${BENTO_HEALTHCHECK_INTERVAL} + start_period: ${BENTO_HEALTHCHECK_START_PERIOD} + start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL} + + tds-db: + image: ${BENTO_TDS_DB_IMAGE}:${BENTO_TDS_DB_VERSION} + container_name: ${BENTO_TDS_DB_CONTAINER_NAME} + networks: + - tds-db-net + environment: + - POSTGRES_USER=${BENTO_TDS_DB_USER} + - POSTGRES_PASSWORD=${BENTO_TDS_DB_PASSWORD} + - POSTGRES_DB=${BENTO_TDS_DB_NAME} + volumes: + - ${BENTO_TDS_DB_VOL_DIR}:/var/lib/postgresql/data + healthcheck: + test: [ "CMD", "pg_isready", "-q", "-d", "${BENTO_TDS_DB_NAME}", "-U", "${BENTO_TDS_DB_USER}" ] + timeout: ${BENTO_HEALTHCHECK_TIMEOUT} + # interval: ${BENTO_HEALTHCHECK_INTERVAL} + # For Docker <25 and Compose <2.24.x, start_interval doesn't work - use a shorter interval for now + interval: ${BENTO_HEALTHCHECK_START_INTERVAL} + start_period: ${BENTO_HEALTHCHECK_START_PERIOD} + start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL} + +networks: + tds-net: + external: true + name: ${BENTO_TDS_NETWORK} + tds-db-net: + external: true + name: ${BENTO_TDS_DB_NETWORK} + + diff --git a/py_bentoctl/other_helpers.py b/py_bentoctl/other_helpers.py index 0d4d87cf..25696a32 100644 --- a/py_bentoctl/other_helpers.py +++ b/py_bentoctl/other_helpers.py @@ -216,6 +216,7 @@ def init_dirs(): "reference-db": "BENTO_REFERENCE_DB_VOL_DIR", "wes": "BENTOV2_WES_VOL_DIR", "wes-tmp": "BENTOV2_WES_VOL_TMP_DIR", + "tds-db": "BENTO_TDS_DB_VOL_DIR", # Feature-specific volume dirs - only if the relevant feature is enabled. # - internal IdP @@ -299,6 +300,8 @@ def init_docker(client: docker.DockerClient): ("BENTO_SERVICE_REGISTRY_NETWORK", dict(driver="bridge")), ("BENTO_WEB_NETWORK", dict(driver="bridge")), ("BENTO_WES_NETWORK", dict(driver="bridge")), + ("BENTO_TDS_NETWORK", dict(driver="bridge")), + ("BENTO_TDS_DB_NETWORK", dict(driver="bridge", internal=True)), # Does not need to access the web ) for net_var, net_kwargs in networks: diff --git a/py_bentoctl/services.py b/py_bentoctl/services.py index b721d499..883ce8bb 100644 --- a/py_bentoctl/services.py +++ b/py_bentoctl/services.py @@ -35,6 +35,7 @@ "katsu-db", "redis", "reference-db", + "tds-db", ) @@ -91,6 +92,8 @@ def _get_service_specific_compose(service: str): "reference-db": ("BENTO_REFERENCE_DB_BASE_IMAGE", "BENTO_REFERENCE_DB_BASE_IMAGE_VERSION", None), "service-registry": ("BENTOV2_SERVICE_REGISTRY_IMAGE", "BENTOV2_SERVICE_REGISTRY_VERSION", "BENTOV2_SERVICE_REGISTRY_VERSION_DEV"), + "tds": ("BENTO_TDS_IMAGE", "BENTO_TDS_VERSION", "BENTO_TDS_VERSION_DEV"), + "tds-db": ("BENTO_TDS_DB_IMAGE", "BENTO_TDS_DB_VERSION", None), "web": ("BENTOV2_WEB_IMAGE", "BENTOV2_WEB_VERSION", "BENTOV2_WEB_VERSION_DEV"), "wes": ("BENTOV2_WES_IMAGE", "BENTOV2_WES_VERSION", "BENTOV2_WES_VERSION_DEV"), }