feat: tds

.gitignore

@@ -51,6 +51,9 @@ lib/public/*
 lib/gohan/es_jvm_options/*
 !lib/gohan/es_jvm_options/.gitkeep
 
+# TDS
+lib/tds/plugins/*
+
 # temp test stuff
 etc/tests/*/__pycache__
 etc/tests/*/*/__pycache__

docker-compose.dev.yaml

@@ -85,6 +85,11 @@ services:
           - ${BENTOV2_DOMAIN}
           - ${BENTOV2_PORTAL_DOMAIN}
           - ${BENTOV2_AUTH_DOMAIN}
+      tds-net:
+        aliases:
+          - ${BENTOV2_DOMAIN}
+          - ${BENTOV2_PORTAL_DOMAIN}
+          - ${BENTOV2_AUTH_DOMAIN}
       web-net:
         aliases:
           - ${BENTOV2_DOMAIN}
@@ -224,6 +229,19 @@ services:
     ports:
       - "${BENTO_REFERENCE_DB_EXTERNAL_PORT}:5432"
 
+  tds:
+    environment:
+      - BENTO_DEBUG=True
+      - BENTO_VALIDATE_SSL=False
+      - LOG_LEVEL=debug
+    ports:
+      - "${BENTO_TDS_EXTERNAL_PORT}:${BENTO_TDS_INTERNAL_PORT}"
+      - "${BENTO_TDS_DEBUGGER_EXTERNAL_PORT}:${BENTO_TDS_DEBUGGER_INTERNAL_PORT}"
+
+  tds-db:
+    ports:
+      - "${BENTO_TDS_DB_EXTERNAL_PORT}:5432"
+
   gohan-elasticsearch:
     # In dev / on Mac, sometimes this will trip up and decide there isn't enough disk space even when there is.
     # See https://stackoverflow.com/questions/48119189/elasticsearch-6-index-change-to-read-only-after-few-second

docker-compose.local.yaml

@@ -156,3 +156,13 @@ services:
     volumes:
       - ./repos/gohan-api:/gohan-api
       - ./repos/gohan-api/src/api/workflows:/app/workflows
+
+  tds:
+    image: ${BENTO_TDS_IMAGE}:${BENTO_TDS_VERSION_DEV}
+    environment:
+      - BENTO_GIT_NAME
+      - BENTO_GIT_EMAIL
+      - BENTO_GIT_REPOSITORY_DIR=/tds
+    volumes:
+      - ./repos/tds:/tds
+

docker-compose.yaml

@@ -19,6 +19,7 @@ include:
       - lib/redis/docker-compose.redis.yaml
       - lib/reference/docker-compose.reference.yaml
       - lib/service-registry/docker-compose.service-registry.yaml
+      - lib/tds/docker-compose.tds.yaml
       - lib/web/docker-compose.web.yaml
       - lib/wes/docker-compose.wes.yaml
     project_directory: .  # Paths in the lib/* compose files must be relative to the Bento base directory

docs/development.md

@@ -140,6 +140,8 @@ The following is a list of all host port allocations for Bento services in devel
 | Reference        | 9510 | 9501          |
 | Reference DB     | 9512 | `N/A`         |
 | Service Registry | 5010 | Unimplemented |
+| TDS              | 9520 | 9520          |
+| TDS DB           | 9522 | `N/A`         |
 | WES              | 9250 | 5680          |
 
 

etc/bento.env

@@ -204,6 +204,29 @@ BENTO_REFERENCE_DB_NAME="reference"
 BENTO_REFERENCE_DB_USER="reference_user"
 # BENTO_REFERENCE_DB_PASSWORD is set in local.env
 
+# Transcriptomics Data Service
+# Service:
+BENTO_TDS_IMAGE=ghcr.io/bento-platform/transcriptomics_data_service
+BENTO_TDS_VERSION=pr-8
+BENTO_TDS_VERSION_DEV=${BENTO_TDS_VERSION}-dev
+BENTO_TDS_CONTAINER_NAME=${BENTOV2_PREFIX}-tds
+BENTO_TDS_NETWORK=${BENTOV2_PREFIX}-tds-net
+BENTO_TDS_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT}
+BENTO_TDS_EXTERNAL_PORT=9520
+BENTO_TDS_DEBUGGER_INTERNAL_PORT=9511
+BENTO_TDS_DEBUGGER_EXTERNAL_PORT=9511
+BENTO_TDS_PLUGINS_DIR=${PWD}/lib/tds/plugins
+# Database
+BENTO_TDS_DB_IMAGE=postgres
+BENTO_TDS_DB_VERSION=16
+BENTO_TDS_DB_CONTAINER_NAME=${BENTOV2_PREFIX}-tds-db
+BENTO_TDS_DB_NETWORK=${BENTOV2_PREFIX}-tds-db-net
+BENTO_TDS_DB_VOL_DIR=${BENTO_FAST_DATA_DIR}/tds/data
+BENTO_TDS_DB_EXTERNAL_PORT=9522
+BENTO_TDS_DB_NAME="tds"
+BENTO_TDS_DB_USER="tds_user"
+# BENTO_TDS_DB_PASSWORD is set in local.env
+
 # WES
 BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes
 BENTOV2_WES_VERSION=0.14.5

etc/bento_services.json

@@ -71,5 +71,10 @@
   },
   "gateway": {
     "repository": "[email protected]:bento-platform/bento_gateway.git"
+  },
+  "tds": {
+    "service_kind": "tds",
+    "url_template": "{BENTO_PORTAL_PUBLIC_URL}/api/{service_kind}",
+    "repository": "[email protected]:bento-platform/transcriptomics_data_service.git"
   }
 }

lib/gateway/docker-compose.gateway.yaml

@@ -78,6 +78,8 @@ services:
       - BENTO_CBIOPORTAL_CONTAINER_NAME
       - BENTO_CBIOPORTAL_INTERNAL_PORT
       - BENTO_GRAFANA_CONTAINER_NAME
+      - BENTO_TDS_CONTAINER_NAME
+      - BENTO_TDS_INTERNAL_PORT
     networks:
       - aggregation-net
       - auth-net
@@ -94,6 +96,7 @@ services:
       - public-net
       - reference-net
       - service-registry-net
+      - tds-net
       - web-net
       - wes-net
     ports:
@@ -186,6 +189,9 @@ networks:
   service-registry-net:
     external: true
     name: ${BENTO_SERVICE_REGISTRY_NETWORK}
+  tds-net:
+    external: true
+    name: ${BENTO_TDS_NETWORK}
   web-net:
     external: true
     name: ${BENTO_WEB_NETWORK}

lib/gateway/services/tds.conf.tpl

@@ -0,0 +1,15 @@
+location /api/tds { return 302 https://${BENTOV2_PORTAL_DOMAIN}/api/tds/; }
+location /api/tds/ {
+    # Reverse proxy settings
+    include /gateway/conf/proxy.conf;
+    include /gateway/conf/proxy_extra.conf;
+
+    # Forward request to TDS
+    rewrite ^ $request_uri;
+    rewrite ^/api/tds/(.*) /$1 break;
+    return 400;
+    proxy_pass  http://${BENTO_TDS_CONTAINER_NAME}:${BENTO_TDS_INTERNAL_PORT}$uri;
+
+    # Errors
+    error_log /var/log/bento_tds_errors.log;
+}

lib/tds/docker-compose.tds.yaml

@@ -0,0 +1,59 @@
+services:
+  tds:
+    image: ${BENTO_TDS_IMAGE}:${BENTO_TDS_VERSION}
+    container_name: ${BENTO_TDS_CONTAINER_NAME}
+    networks:
+      - tds-net
+      - tds-db-net
+    expose:
+      - ${BENTO_TDS_INTERNAL_PORT}
+    depends_on:
+      tds-db:
+        condition: service_healthy
+    environment:
+      - BENTO_UID
+      - BENTO_DEBUG=False
+      - DATABASE_URI=postgres://${BENTO_TDS_DB_USER}:${BENTO_TDS_DB_PASSWORD}@${BENTO_TDS_DB_CONTAINER_NAME}:5432/${BENTO_TDS_DB_NAME}
+      - INTERNAL_PORT=${BENTO_TDS_INTERNAL_PORT}
+      - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/tds
+      - CORS_ORIGINS=${BENTO_CORS_ORIGINS}
+      - BENTO_AUTHZ_SERVICE_URL
+      - BENTO_AUTHZ_ENABLED=True
+    volumes:
+      - ${BENTO_TDS_PLUGINS_DIR}:/tds/lib
+    healthcheck:
+      test: ["CMD", "curl", "http://localhost:${BENTO_TDS_INTERNAL_PORT}/service-info"]
+      timeout: ${BENTO_HEALTHCHECK_TIMEOUT}
+      interval: ${BENTO_HEALTHCHECK_INTERVAL}
+      start_period: ${BENTO_HEALTHCHECK_START_PERIOD}
+      start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL}
+
+  tds-db:
+    image: ${BENTO_TDS_DB_IMAGE}:${BENTO_TDS_DB_VERSION}
+    container_name: ${BENTO_TDS_DB_CONTAINER_NAME}
+    networks:
+      - tds-db-net
+    environment:
+      - POSTGRES_USER=${BENTO_TDS_DB_USER}
+      - POSTGRES_PASSWORD=${BENTO_TDS_DB_PASSWORD}
+      - POSTGRES_DB=${BENTO_TDS_DB_NAME}
+    volumes:
+      - ${BENTO_TDS_DB_VOL_DIR}:/var/lib/postgresql/data
+    healthcheck:
+      test: [ "CMD", "pg_isready", "-q", "-d", "${BENTO_TDS_DB_NAME}", "-U", "${BENTO_TDS_DB_USER}" ]
+      timeout: ${BENTO_HEALTHCHECK_TIMEOUT}
+      # interval: ${BENTO_HEALTHCHECK_INTERVAL}
+      # For Docker <25 and Compose <2.24.x, start_interval doesn't work - use a shorter interval for now
+      interval: ${BENTO_HEALTHCHECK_START_INTERVAL}
+      start_period: ${BENTO_HEALTHCHECK_START_PERIOD}
+      start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL}
+
+networks:
+  tds-net:
+    external: true
+    name: ${BENTO_TDS_NETWORK}
+  tds-db-net:
+    external: true
+    name: ${BENTO_TDS_DB_NETWORK}
+
+

py_bentoctl/other_helpers.py

@@ -216,6 +216,7 @@ def init_dirs():
         "reference-db": "BENTO_REFERENCE_DB_VOL_DIR",
         "wes": "BENTOV2_WES_VOL_DIR",
         "wes-tmp": "BENTOV2_WES_VOL_TMP_DIR",
+        "tds-db": "BENTO_TDS_DB_VOL_DIR",
 
         # Feature-specific volume dirs - only if the relevant feature is enabled.
         #  - internal IdP
@@ -299,6 +300,8 @@ def init_docker(client: docker.DockerClient):
         ("BENTO_SERVICE_REGISTRY_NETWORK", dict(driver="bridge")),
         ("BENTO_WEB_NETWORK", dict(driver="bridge")),
         ("BENTO_WES_NETWORK", dict(driver="bridge")),
+        ("BENTO_TDS_NETWORK", dict(driver="bridge")),
+        ("BENTO_TDS_DB_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web
     )
 
     for net_var, net_kwargs in networks:

py_bentoctl/services.py

@@ -35,6 +35,7 @@
     "katsu-db",
     "redis",
     "reference-db",
+    "tds-db",
 )
 
 
@@ -91,6 +92,8 @@ def _get_service_specific_compose(service: str):
     "reference-db": ("BENTO_REFERENCE_DB_BASE_IMAGE", "BENTO_REFERENCE_DB_BASE_IMAGE_VERSION", None),
     "service-registry": ("BENTOV2_SERVICE_REGISTRY_IMAGE", "BENTOV2_SERVICE_REGISTRY_VERSION",
                          "BENTOV2_SERVICE_REGISTRY_VERSION_DEV"),
+    "tds": ("BENTO_TDS_IMAGE", "BENTO_TDS_VERSION", "BENTO_TDS_VERSION_DEV"),
+    "tds-db": ("BENTO_TDS_DB_IMAGE", "BENTO_TDS_DB_VERSION", None),
     "web": ("BENTOV2_WEB_IMAGE", "BENTOV2_WEB_VERSION", "BENTOV2_WEB_VERSION_DEV"),
     "wes": ("BENTOV2_WES_IMAGE", "BENTOV2_WES_VERSION", "BENTOV2_WES_VERSION_DEV"),
 }