DIG-1835: make a self-contained init-minio make target

.gitignore

@@ -50,3 +50,8 @@ pnpm-debug.log*
 
 # macOS-specific files
 .DS_Store
+
+# minio-related files
+lib/minio/access-key
+lib/minio/secret-key
+lib/minio/aws-credentials

Makefile

@@ -259,7 +259,7 @@ clean-secrets:
 
 
 #>>>
-# remove all peristant volumes and local data
+# remove all persistent volumes and local data
 # make clean-volumes
 
 #<<<
@@ -360,7 +360,7 @@ docker-push:
 
 #<<<
 .PHONY: docker-secrets
-docker-secrets: mkdir authx-secrets data-secrets #minio-secrets
+docker-secrets: mkdir authx-secrets data-secrets
 
 
 data-secrets: mkdir
@@ -383,11 +383,12 @@ authx-secrets: mkdir
 
 minio-secrets: mkdir
 	@echo "making minio secrets"
-	@echo $(DEFAULT_ADMIN_USER) > tmp/secrets/minio-access-key
+	@echo $(DEFAULT_ADMIN_USER) > lib/minio/access-key
 	$(MAKE) secret-minio-secret-key
-	@echo '[default]' > tmp/secrets/aws-credentials
-	@echo "aws_access_key_id=`cat tmp/secrets/minio-access-key`" >> tmp/secrets/aws-credentials
-	@echo "aws_secret_access_key=`cat tmp/secrets/minio-secret-key`" >> tmp/secrets/aws-credentials
+	mv tmp/secrets/minio-secret-key lib/minio/secret-key
+	@echo '[default]' > lib/minio/aws-credentials
+	@echo "aws_access_key_id=`cat lib/minio/access-key`" >> lib/minio/aws-credentials
+	@echo "aws_secret_access_key=`cat lib/minio/secret-key`" >> lib/minio/aws-credentials
 
 
 #>>>
@@ -399,8 +400,6 @@ minio-secrets: mkdir
 docker-volumes:
 	docker volume create grafana-data --label candigv2=volume
 	docker volume create jupyter-data --label candigv2=volume
-	# docker volume create minio-config --label candigv2=volume
-	# docker volume create minio-data $(MINIO_VOLUME_OPT) --label candigv2=volume
 	docker volume create prometheus-data --label candigv2=volume
 	docker volume create toil-jobstore --label candigv2=volume
 	docker volume create keycloak-data --label candigv2=volume
@@ -425,6 +424,17 @@ init-authx: mkdir
 	$(foreach MODULE, $(CANDIG_AUTH_MODULES), $(MAKE) build-$(MODULE); $(MAKE) compose-$(MODULE); python settings.py;)
 
 
+#>>>
+# create a minio container (that won't be removed as part of clean-all)
+# make init-minio
+
+#<<<
+init-minio: minio-secrets
+	docker volume create minio-config
+	docker volume create minio-data $(MINIO_VOLUME_OPT)
+	docker compose -f lib/candigv2/docker-compose.yml -f lib/minio/docker-compose.yml --compatibility up -d 2>&1 | tee -a $(ERRORLOG)
+
+
 #>>>
 # initialize conda environment
 # make init-conda

etc/env/example.env

@@ -3,7 +3,7 @@
 
 # site options
 CANDIG_MODULES=logging keycloak vault redis postgres htsget katsu query tyk opa federation candig-ingest candig-data-portal
-    #minio drs-server wes-server monitoring
+    #drs-server wes-server monitoring
 CANDIG_AUTH_MODULES=keycloak vault tyk opa federation
 CANDIG_DATA_MODULES=keycloak vault redis postgres logging
 

lib/candigv2/docker-compose.yml

@@ -1,8 +1,4 @@
 volumes:
-  # minio-data:
-  #   external: true
-  # minio-config:
-  #   external: true
   toil-jobstore:
     external: true
   prometheus-data:
@@ -31,14 +27,6 @@ secrets:
     file: $PWD/tmp/postgres/db-secret
     labels:
       - "candigv2=secret"
-  # minio-access-key:
-  #   file: $PWD/tmp/secrets/minio-access-key
-  #   labels:
-  #     - "candigv2=secret"
-  # minio-secret-key:
-  #   file: $PWD/tmp/secrets/minio-secret-key
-  #   labels:
-  #     - "candigv2=secret"
   wes-dependency-resolver:
     file: $PWD/etc/yml/${WES_DEPENDENCY_RESOLVER}.yml
     labels:

lib/minio/docker-compose.yml

@@ -1,8 +1,16 @@
+volumes:
+  minio-data:
+    external: true
+  minio-config:
+    external: true
+secrets:
+  minio-access-key:
+    file: $PWD/lib/minio/access-key
+  minio-secret-key:
+    file: $PWD/lib/minio/secret-key
 services:
   minio:
     image: minio/minio:${MINIO_VERSION:-latest}
-    labels:
-      - "candigv2=minio"
     volumes:
       - minio-data:/data
       - minio-config:/root/.minio

lib/toil/docker-compose.yml

@@ -1,3 +1,10 @@
+secrets:
+  minio-access-key:
+    file: $PWD/lib/minio/access-key
+  minio-secret-key:
+    file: $PWD/lib/minio/secret-key
+  aws-credentials:
+    file: $PWD/lib/minio/aws-credentials
 services:
   toil-server:
     image: ${DOCKER_REGISTRY}/toil:${TOIL_VERSION:-latest}

lib/wes-server/docker-compose.yml

@@ -1,3 +1,10 @@
+secrets:
+  minio-access-key:
+    file: $PWD/lib/minio/access-key
+  minio-secret-key:
+    file: $PWD/lib/minio/secret-key
+  aws-credentials:
+    file: $PWD/lib/minio/aws-credentials
 services:
   wes-server:
     build: