From fc1ba5fd86cff8b10bc1e1a1041d052f37e32a96 Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Wed, 20 Nov 2024 17:30:03 -0800 Subject: [PATCH 1/3] make a self-container init-minio make target --- Makefile | 11 +++++++++++ etc/env/example.env | 2 +- lib/minio/docker-compose.yml | 12 ++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7b94d2cbd..5d216ba08 100644 --- a/Makefile +++ b/Makefile @@ -425,6 +425,17 @@ init-authx: mkdir $(foreach MODULE, $(CANDIG_AUTH_MODULES), $(MAKE) build-$(MODULE); $(MAKE) compose-$(MODULE); python settings.py;) +#>>> +# create a minio container (that won't be removed as part of clean-all) +# make init-minio + +#<<< +init-minio: minio-secrets + docker volume create minio-config + docker volume create minio-data $(MINIO_VOLUME_OPT) + docker compose -f lib/candigv2/docker-compose.yml -f lib/minio/docker-compose.yml --compatibility up -d 2>&1 | tee -a $(ERRORLOG) + + #>>> # initialize conda environment # make init-conda diff --git a/etc/env/example.env b/etc/env/example.env index aa3dfb893..aaa6fcc7b 100644 --- a/etc/env/example.env +++ b/etc/env/example.env @@ -3,7 +3,7 @@ # site options CANDIG_MODULES=logging keycloak vault redis postgres htsget katsu query tyk opa federation candig-ingest candig-data-portal - #minio drs-server wes-server monitoring + #drs-server wes-server monitoring CANDIG_AUTH_MODULES=keycloak vault tyk opa federation CANDIG_DATA_MODULES=keycloak vault redis postgres logging diff --git a/lib/minio/docker-compose.yml b/lib/minio/docker-compose.yml index fbb70ae37..6f2c675ab 100644 --- a/lib/minio/docker-compose.yml +++ b/lib/minio/docker-compose.yml @@ -1,8 +1,16 @@ +volumes: + minio-data: + external: true + minio-config: + external: true +secrets: + minio-access-key: + file: $PWD/tmp/secrets/minio-access-key + minio-secret-key: + file: $PWD/tmp/secrets/minio-secret-key services: minio: image: minio/minio:${MINIO_VERSION:-latest} - labels: - - "candigv2=minio" volumes: - minio-data:/data - minio-config:/root/.minio From b09c6ed8131bfc4930b15798c2cbae258d6fcb55 Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Wed, 20 Nov 2024 17:31:55 -0800 Subject: [PATCH 2/3] remove commented-out bits --- Makefile | 4 +--- lib/candigv2/docker-compose.yml | 12 ------------ 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 5d216ba08..064228cea 100644 --- a/Makefile +++ b/Makefile @@ -360,7 +360,7 @@ docker-push: #<<< .PHONY: docker-secrets -docker-secrets: mkdir authx-secrets data-secrets #minio-secrets +docker-secrets: mkdir authx-secrets data-secrets data-secrets: mkdir @@ -399,8 +399,6 @@ minio-secrets: mkdir docker-volumes: docker volume create grafana-data --label candigv2=volume docker volume create jupyter-data --label candigv2=volume - # docker volume create minio-config --label candigv2=volume - # docker volume create minio-data $(MINIO_VOLUME_OPT) --label candigv2=volume docker volume create prometheus-data --label candigv2=volume docker volume create toil-jobstore --label candigv2=volume docker volume create keycloak-data --label candigv2=volume diff --git a/lib/candigv2/docker-compose.yml b/lib/candigv2/docker-compose.yml index d9aa723d5..4669f8547 100644 --- a/lib/candigv2/docker-compose.yml +++ b/lib/candigv2/docker-compose.yml @@ -1,8 +1,4 @@ volumes: - # minio-data: - # external: true - # minio-config: - # external: true toil-jobstore: external: true prometheus-data: @@ -31,14 +27,6 @@ secrets: file: $PWD/tmp/postgres/db-secret labels: - "candigv2=secret" - # minio-access-key: - # file: $PWD/tmp/secrets/minio-access-key - # labels: - # - "candigv2=secret" - # minio-secret-key: - # file: $PWD/tmp/secrets/minio-secret-key - # labels: - # - "candigv2=secret" wes-dependency-resolver: file: $PWD/etc/yml/${WES_DEPENDENCY_RESOLVER}.yml labels: From d9fe8470f180bf008287b26765fca94590be90ce Mon Sep 17 00:00:00 2001 From: Daisie Huang Date: Thu, 21 Nov 2024 11:37:12 -0800 Subject: [PATCH 3/3] move secrets around --- .gitignore | 5 +++++ Makefile | 11 ++++++----- lib/minio/docker-compose.yml | 4 ++-- lib/toil/docker-compose.yml | 7 +++++++ lib/wes-server/docker-compose.yml | 7 +++++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 5d8e035ba..fd03f5829 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,8 @@ pnpm-debug.log* # macOS-specific files .DS_Store + +# minio-related files +lib/minio/access-key +lib/minio/secret-key +lib/minio/aws-credentials diff --git a/Makefile b/Makefile index 064228cea..1017e4694 100644 --- a/Makefile +++ b/Makefile @@ -259,7 +259,7 @@ clean-secrets: #>>> -# remove all peristant volumes and local data +# remove all persistent volumes and local data # make clean-volumes #<<< @@ -383,11 +383,12 @@ authx-secrets: mkdir minio-secrets: mkdir @echo "making minio secrets" - @echo $(DEFAULT_ADMIN_USER) > tmp/secrets/minio-access-key + @echo $(DEFAULT_ADMIN_USER) > lib/minio/access-key $(MAKE) secret-minio-secret-key - @echo '[default]' > tmp/secrets/aws-credentials - @echo "aws_access_key_id=`cat tmp/secrets/minio-access-key`" >> tmp/secrets/aws-credentials - @echo "aws_secret_access_key=`cat tmp/secrets/minio-secret-key`" >> tmp/secrets/aws-credentials + mv tmp/secrets/minio-secret-key lib/minio/secret-key + @echo '[default]' > lib/minio/aws-credentials + @echo "aws_access_key_id=`cat lib/minio/access-key`" >> lib/minio/aws-credentials + @echo "aws_secret_access_key=`cat lib/minio/secret-key`" >> lib/minio/aws-credentials #>>> diff --git a/lib/minio/docker-compose.yml b/lib/minio/docker-compose.yml index 6f2c675ab..af66340fd 100644 --- a/lib/minio/docker-compose.yml +++ b/lib/minio/docker-compose.yml @@ -5,9 +5,9 @@ volumes: external: true secrets: minio-access-key: - file: $PWD/tmp/secrets/minio-access-key + file: $PWD/lib/minio/access-key minio-secret-key: - file: $PWD/tmp/secrets/minio-secret-key + file: $PWD/lib/minio/secret-key services: minio: image: minio/minio:${MINIO_VERSION:-latest} diff --git a/lib/toil/docker-compose.yml b/lib/toil/docker-compose.yml index afbef2209..c65008485 100644 --- a/lib/toil/docker-compose.yml +++ b/lib/toil/docker-compose.yml @@ -1,3 +1,10 @@ +secrets: + minio-access-key: + file: $PWD/lib/minio/access-key + minio-secret-key: + file: $PWD/lib/minio/secret-key + aws-credentials: + file: $PWD/lib/minio/aws-credentials services: toil-server: image: ${DOCKER_REGISTRY}/toil:${TOIL_VERSION:-latest} diff --git a/lib/wes-server/docker-compose.yml b/lib/wes-server/docker-compose.yml index 0d858e112..72943f6ff 100644 --- a/lib/wes-server/docker-compose.yml +++ b/lib/wes-server/docker-compose.yml @@ -1,3 +1,10 @@ +secrets: + minio-access-key: + file: $PWD/lib/minio/access-key + minio-secret-key: + file: $PWD/lib/minio/secret-key + aws-credentials: + file: $PWD/lib/minio/aws-credentials services: wes-server: build: