From 9cfffff1c33824df67dc7386e27df29338192ddd Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 13 Aug 2024 17:35:26 +0200 Subject: [PATCH 1/9] Added: localstack container and related config to environment --- dev-env/conf/localstack/buckets.sh | 3 +++ dev-env/docker-compose-dev.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 dev-env/conf/localstack/buckets.sh diff --git a/dev-env/conf/localstack/buckets.sh b/dev-env/conf/localstack/buckets.sh new file mode 100644 index 000000000..60f44f7aa --- /dev/null +++ b/dev-env/conf/localstack/buckets.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack +awslocal s3 mb s3://mybucket \ No newline at end of file diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml index 68b1b6c40..76fee694f 100644 --- a/dev-env/docker-compose-dev.yml +++ b/dev-env/docker-compose-dev.yml @@ -54,6 +54,17 @@ services: -Ddataverse.pid.fake.label=FakeDOIProvider -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ + -Ddataverse.files.storage-driver-id=localstack1 + -Ddataverse.files.localstack1.type=s3 + -Ddataverse.files.localstack1.label=LocalStack + -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566 + -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2 + -Ddataverse.files.localstack1.bucket-name=mybucket + -Ddataverse.files.localstack1.path-style-access=true + -Ddataverse.files.localstack1.upload-redirect=true + -Ddataverse.files.localstack1.download-redirect=true + -Ddataverse.files.localstack1.access-key=default + -Ddataverse.files.localstack1.secret-key=default expose: - '8080' networks: @@ -61,6 +72,7 @@ services: depends_on: - dev_postgres - dev_solr + - dev_localstack volumes: - ./docker-dev-volumes/app/data:/dv - ./docker-dev-volumes/app/secrets:/secrets @@ -82,6 +94,23 @@ services: volumes: - ./docker-dev-volumes/app/data:/dv + dev_localstack: + container_name: 'dev_localstack' + hostname: 'localstack' + image: localstack/localstack:2.3.2 + restart: on-failure + ports: + - '4566:4566' + environment: + - DOCKER_HOST=unix:///var/run/docker.sock + - HOSTNAME_EXTERNAL=localstack + networks: + - dataverse + volumes: + - ./conf/localstack:/etc/localstack/init/ready.d + tmpfs: + - /localstack:mode=770,size=128M,uid=1000,gid=1000 + dev_dataverse_bootstrap: container_name: 'dev_dataverse_bootstrap' image: gdcc/configbaker:unstable From 9a34e292ca820051e4236f2680a70c2408ddf0b6 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 14 Aug 2024 19:50:54 +0100 Subject: [PATCH 2/9] Added: CORS config to localstack container --- dev-env/conf/localstack/buckets.sh | 10 ++++++++-- dev-env/conf/localstack/cors.json | 11 +++++++++++ dev-env/docker-compose-dev.yml | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) mode change 100644 => 100755 dev-env/conf/localstack/buckets.sh create mode 100644 dev-env/conf/localstack/cors.json diff --git a/dev-env/conf/localstack/buckets.sh b/dev-env/conf/localstack/buckets.sh old mode 100644 new mode 100755 index 60f44f7aa..119a46f69 --- a/dev-env/conf/localstack/buckets.sh +++ b/dev-env/conf/localstack/buckets.sh @@ -1,3 +1,9 @@ #!/usr/bin/env bash -# https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack -awslocal s3 mb s3://mybucket \ No newline at end of file + +echo "Creating S3 bucket..." +awslocal s3 mb s3://mybucket || { echo "Failed to create S3 bucket"; exit 1; } + +echo "Applying CORS configuration..." +awslocal s3api put-bucket-cors --bucket mybucket --cors-configuration file:///etc/localstack/init/ready.d/cors.json || { echo "Failed to apply CORS configuration"; exit 1; } + +echo "CORS configuration applied successfully" diff --git a/dev-env/conf/localstack/cors.json b/dev-env/conf/localstack/cors.json new file mode 100644 index 000000000..380b3b830 --- /dev/null +++ b/dev-env/conf/localstack/cors.json @@ -0,0 +1,11 @@ +{ + "CORSRules": [ + { + "AllowedOrigins": ["http://localhost:8000"], + "AllowedMethods": ["GET", "POST", "PUT", "DELETE", "HEAD"], + "AllowedHeaders": ["*"], + "ExposeHeaders": ["ETag"], + "MaxAgeSeconds": 3000 + } + ] +} diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml index 76fee694f..26c1dfe11 100644 --- a/dev-env/docker-compose-dev.yml +++ b/dev-env/docker-compose-dev.yml @@ -57,7 +57,7 @@ services: -Ddataverse.files.storage-driver-id=localstack1 -Ddataverse.files.localstack1.type=s3 -Ddataverse.files.localstack1.label=LocalStack - -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566 + -Ddataverse.files.localstack1.custom-endpoint-url=http://localhost:4566 -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2 -Ddataverse.files.localstack1.bucket-name=mybucket -Ddataverse.files.localstack1.path-style-access=true From 5ab638c41ea45f2b1f80fb9ac730a9d410fe06c1 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 15 Aug 2024 10:32:03 +0100 Subject: [PATCH 3/9] Fixed: localstack container visibility issue from outside containerized network --- dev-env/docker-compose-dev.yml | 2 +- dev-env/run-env.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml index 26c1dfe11..76fee694f 100644 --- a/dev-env/docker-compose-dev.yml +++ b/dev-env/docker-compose-dev.yml @@ -57,7 +57,7 @@ services: -Ddataverse.files.storage-driver-id=localstack1 -Ddataverse.files.localstack1.type=s3 -Ddataverse.files.localstack1.label=LocalStack - -Ddataverse.files.localstack1.custom-endpoint-url=http://localhost:4566 + -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566 -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2 -Ddataverse.files.localstack1.bucket-name=mybucket -Ddataverse.files.localstack1.path-style-access=true diff --git a/dev-env/run-env.sh b/dev-env/run-env.sh index f8014c429..e4b89f88d 100755 --- a/dev-env/run-env.sh +++ b/dev-env/run-env.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +echo "INFO - Configuring etc/hosts to support direct upload through containers..." + +HOSTS_FILE="/etc/hosts" +HOSTNAME="localstack" +IP_ADDRESS="127.0.0.1" + +add_host_entry() { + if grep -q "$HOSTNAME" "$HOSTS_FILE"; then + echo "$HOSTNAME is already present in $HOSTS_FILE" + else + echo "Adding $HOSTNAME to $HOSTS_FILE" + echo "$IP_ADDRESS $HOSTNAME" | sudo tee -a "$HOSTS_FILE" > /dev/null + echo "$HOSTNAME added successfully" + fi +} + +add_host_entry + export DATAVERSE_IMAGE_TAG=$1 # To avoid timeout issues on frontend container startup From 650cbcf3b355341e7ba53b590106e8503633f40d Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 15 Aug 2024 12:58:30 +0100 Subject: [PATCH 4/9] Changed: use remote s3 aws storage instead of containerized --- dev-env/{.env => .env.example} | 2 ++ dev-env/.gitignore | 1 + dev-env/docker-compose-dev.yml | 42 +++++++++++----------------------- dev-env/run-env.sh | 18 --------------- 4 files changed, 16 insertions(+), 47 deletions(-) rename dev-env/{.env => .env.example} (58%) create mode 100644 dev-env/.gitignore diff --git a/dev-env/.env b/dev-env/.env.example similarity index 58% rename from dev-env/.env rename to dev-env/.env.example index 05d4bebe8..972aa9026 100644 --- a/dev-env/.env +++ b/dev-env/.env.example @@ -2,3 +2,5 @@ POSTGRES_VERSION=13 DATAVERSE_DB_USER=dataverse SOLR_VERSION=9.3.0 REGISTRY=docker.io +S3_ACCESS_KEY= +S3_SECRET_KEY= diff --git a/dev-env/.gitignore b/dev-env/.gitignore new file mode 100644 index 000000000..4c49bd78f --- /dev/null +++ b/dev-env/.gitignore @@ -0,0 +1 @@ +.env diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml index 76fee694f..3c957c651 100644 --- a/dev-env/docker-compose-dev.yml +++ b/dev-env/docker-compose-dev.yml @@ -54,17 +54,19 @@ services: -Ddataverse.pid.fake.label=FakeDOIProvider -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ - -Ddataverse.files.storage-driver-id=localstack1 - -Ddataverse.files.localstack1.type=s3 - -Ddataverse.files.localstack1.label=LocalStack - -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566 - -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2 - -Ddataverse.files.localstack1.bucket-name=mybucket - -Ddataverse.files.localstack1.path-style-access=true - -Ddataverse.files.localstack1.upload-redirect=true - -Ddataverse.files.localstack1.download-redirect=true - -Ddataverse.files.localstack1.access-key=default - -Ddataverse.files.localstack1.secret-key=default + -Ddataverse.files.s3.access-key=${S3_ACCESS_KEY} + -Ddataverse.files.s3.label=s3 + -Ddataverse.files.s3.secret-key=${S3_SECRET_KEY} + -Ddataverse.files.storage-driver-id=s3 + -Ddataverse.files.s3.type=s3 + -Ddataverse.files.s3.bucket-name=beta-dataverse-direct + -Ddataverse.files.s3.upload-redirect=true + -Ddataverse.files.s3.download-redirect=true + -Ddataverse.files.s3.ingestsizelimit=50000000 + -Ddataverse.files.s3.url-expiration-minutes=60 + -Ddataverse.files.s3.connection-pool-size=2048 + -Ddataverse.files.s3.custom-endpoint-region=us-east-1 + -Ddataverse.files.s3.custom-endpoint-url=https://s3.us-east-1.amazonaws.com expose: - '8080' networks: @@ -72,7 +74,6 @@ services: depends_on: - dev_postgres - dev_solr - - dev_localstack volumes: - ./docker-dev-volumes/app/data:/dv - ./docker-dev-volumes/app/secrets:/secrets @@ -94,23 +95,6 @@ services: volumes: - ./docker-dev-volumes/app/data:/dv - dev_localstack: - container_name: 'dev_localstack' - hostname: 'localstack' - image: localstack/localstack:2.3.2 - restart: on-failure - ports: - - '4566:4566' - environment: - - DOCKER_HOST=unix:///var/run/docker.sock - - HOSTNAME_EXTERNAL=localstack - networks: - - dataverse - volumes: - - ./conf/localstack:/etc/localstack/init/ready.d - tmpfs: - - /localstack:mode=770,size=128M,uid=1000,gid=1000 - dev_dataverse_bootstrap: container_name: 'dev_dataverse_bootstrap' image: gdcc/configbaker:unstable diff --git a/dev-env/run-env.sh b/dev-env/run-env.sh index e4b89f88d..f8014c429 100755 --- a/dev-env/run-env.sh +++ b/dev-env/run-env.sh @@ -1,23 +1,5 @@ #!/usr/bin/env bash -echo "INFO - Configuring etc/hosts to support direct upload through containers..." - -HOSTS_FILE="/etc/hosts" -HOSTNAME="localstack" -IP_ADDRESS="127.0.0.1" - -add_host_entry() { - if grep -q "$HOSTNAME" "$HOSTS_FILE"; then - echo "$HOSTNAME is already present in $HOSTS_FILE" - else - echo "Adding $HOSTNAME to $HOSTS_FILE" - echo "$IP_ADDRESS $HOSTNAME" | sudo tee -a "$HOSTS_FILE" > /dev/null - echo "$HOSTNAME added successfully" - fi -} - -add_host_entry - export DATAVERSE_IMAGE_TAG=$1 # To avoid timeout issues on frontend container startup From 1f29b7cb78951b03461a38910672b27d25d96d08 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 15 Aug 2024 12:59:56 +0100 Subject: [PATCH 5/9] Added: action npmrc setup and cleanup --- .github/workflows/test.yml | 6 ++++++ dev-env/conf/localstack/buckets.sh | 9 --------- dev-env/conf/localstack/cors.json | 11 ----------- 3 files changed, 6 insertions(+), 20 deletions(-) delete mode 100755 dev-env/conf/localstack/buckets.sh delete mode 100644 dev-env/conf/localstack/cors.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db4356d24..3defb0575 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,12 @@ jobs: sed -i~ '/^REGISTRY=/s/=.*/=docker.io/' .env shell: bash + - name: Create .npmrc + run: | + cp .env.example .env + sed -i -e 's//${{ secrets.S3_ACCESS_KEY }}/g' .env + sed -i -e 's//${{ secrets.S3_SECRET_KEY }}/g' .env + - name: Start containers working-directory: dev-env run: ./run-env.sh "$E2E_DATAVERSE_IMAGE_TAG" diff --git a/dev-env/conf/localstack/buckets.sh b/dev-env/conf/localstack/buckets.sh deleted file mode 100755 index 119a46f69..000000000 --- a/dev-env/conf/localstack/buckets.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -echo "Creating S3 bucket..." -awslocal s3 mb s3://mybucket || { echo "Failed to create S3 bucket"; exit 1; } - -echo "Applying CORS configuration..." -awslocal s3api put-bucket-cors --bucket mybucket --cors-configuration file:///etc/localstack/init/ready.d/cors.json || { echo "Failed to apply CORS configuration"; exit 1; } - -echo "CORS configuration applied successfully" diff --git a/dev-env/conf/localstack/cors.json b/dev-env/conf/localstack/cors.json deleted file mode 100644 index 380b3b830..000000000 --- a/dev-env/conf/localstack/cors.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "CORSRules": [ - { - "AllowedOrigins": ["http://localhost:8000"], - "AllowedMethods": ["GET", "POST", "PUT", "DELETE", "HEAD"], - "AllowedHeaders": ["*"], - "ExposeHeaders": ["ETag"], - "MaxAgeSeconds": 3000 - } - ] -} From 030b8f4090ae150c423c427f9f0550cb6646f023 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 15 Aug 2024 13:03:58 +0100 Subject: [PATCH 6/9] Fixed: action step --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3defb0575..2d2e904ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,6 +48,7 @@ jobs: shell: bash - name: Create .npmrc + working-directory: dev-env run: | cp .env.example .env sed -i -e 's//${{ secrets.S3_ACCESS_KEY }}/g' .env From 429c39acd2557726c5fa117c118953f9d075e988 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 15 Aug 2024 13:14:44 +0100 Subject: [PATCH 7/9] Fixed: test action steps --- .github/workflows/test.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d2e904ae..848e3fa76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,18 +41,23 @@ jobs: echo VITE_DATAVERSE_BACKEND_URL="$DATAVERSE_BACKEND_URL" >> .env shell: bash - - name: Update registry for the containerized development environment + - name: Create containerized development environment .env file working-directory: dev-env - run: | - sed -i~ '/^REGISTRY=/s/=.*/=docker.io/' .env + run: cp .env.example .env shell: bash - - name: Create .npmrc + - name: Set S3 secrets for the containerized development environment working-directory: dev-env run: | - cp .env.example .env sed -i -e 's//${{ secrets.S3_ACCESS_KEY }}/g' .env sed -i -e 's//${{ secrets.S3_SECRET_KEY }}/g' .env + shell: bash + + - name: Update registry for the containerized development environment + working-directory: dev-env + run: | + sed -i~ '/^REGISTRY=/s/=.*/=docker.io/' .env + shell: bash - name: Start containers working-directory: dev-env From 5abff2bd2530d0df727cc2e7ed20ebc97cf59d00 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 16 Aug 2024 11:50:12 +0100 Subject: [PATCH 8/9] Changed: s3 as default storage option --- dev-env/docker-compose-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml index 3c957c651..94b83bae0 100644 --- a/dev-env/docker-compose-dev.yml +++ b/dev-env/docker-compose-dev.yml @@ -44,6 +44,7 @@ services: restart: on-failure user: payara environment: + dataverse_files_storage__driver__id: s3 DATAVERSE_DB_HOST: postgres DATAVERSE_DB_PASSWORD: secret DATAVERSE_DB_USER: ${DATAVERSE_DB_USER} From 0ca4a1f1067a8abd3ff421727d31b467f6efd0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20Saracca?= Date: Wed, 21 Aug 2024 08:20:50 -0300 Subject: [PATCH 9/9] fix: missing type prop on LinkToPage --- .../collection/datasets-list/file-card/FileCardHeader.tsx | 2 ++ .../collection/datasets-list/file-card/FileCardInfo.tsx | 2 ++ .../collection/datasets-list/file-card/FileCardThumbnail.tsx | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/sections/collection/datasets-list/file-card/FileCardHeader.tsx b/src/sections/collection/datasets-list/file-card/FileCardHeader.tsx index 034e7d27b..a8a887920 100644 --- a/src/sections/collection/datasets-list/file-card/FileCardHeader.tsx +++ b/src/sections/collection/datasets-list/file-card/FileCardHeader.tsx @@ -6,6 +6,7 @@ import { DatasetLabels } from '../../../dataset/dataset-labels/DatasetLabels' import { FileCardIcon } from './FileCardIcon' import { FileType } from '../../../../files/domain/models/FileMetadata' import { FileCardHelper } from './FileCardHelper' +import { DvObjectType } from '../../../../shared/hierarchy/domain/models/UpwardHierarchyNode' interface FileCardHeaderProps { filePreview: FilePreview @@ -18,6 +19,7 @@ export function FileCardHeader({ filePreview }: FileCardHeaderProps) {