From a210696a7d0aeeddcd47471beba9d1ce9b1fc0ef Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:02:37 +0100 Subject: [PATCH 1/8] add documentation about upload/download and fix markdown --- doc/technical-overview.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/technical-overview.md b/doc/technical-overview.md index d37040fb7..606e6ad45 100644 --- a/doc/technical-overview.md +++ b/doc/technical-overview.md @@ -453,8 +453,8 @@ Personnel have the permissions to upload data. When starting an upload, a directory (“staging directory”) is created by the executing command. The default location of the staging directory is the current working directory, however the user can specify an existing directory in which the staging directory should be placed. Independent of the location -(specified or default), the staging directory is named _DataDelivery**_, -where is the date and time when the upload was started, and is the ID of +(specified or default), the staging directory is named _DataDelivery\_\\_\\_upload_, +where \ is the date and time when the upload was started, and \ is the ID of the project the user is attempting to upload data to. If there is no data to upload, this directory is deleted immediately. If not, the staging directory will contain three subdirectories: @@ -533,7 +533,7 @@ When downloading data, the Researchers can either choose to download specific fi folder(s), or the entire project contents. As with the upload ([Uploading data](#uploading-data)), a staging directory is created when downloading the data. This -directory is placed by default in the current working directory, and is named DataDelivery**. +directory is placed by default in the current working directory, and is named _DataDelivery\_\\_\\_download_. However unlike the upload command, downloading allows the user to choose the name of the directory - specifying a destination. The destination cannot be an existing directory[^22] - it must be a new directory. Since a new destination is required with every download, downloading the same file(s) multiple times is possible and is only limited by the amount From c7eb80d5dd916b1623b90aed155e2275ad0c891f Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 11 Dec 2024 16:05:56 +0100 Subject: [PATCH 2/8] try to fix tests --- docker-compose.yml | 2 +- tests/conftest.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7cda25b4a..1df613bfa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ version: "3.9" services: db: container_name: dds_database - image: mariadb:10.11.5 + image: mariadb:11.6.2 environment: - MYSQL_ROOT_PASSWORD=${DDS_MYSQL_ROOT_PASS} - MYSQL_USER=${DDS_MYSQL_USER} diff --git a/tests/conftest.py b/tests/conftest.py index a9d05709b..e7fa1eab2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,12 +103,49 @@ def fill_basic_db(db): def new_test_db(uri): dbname = uri[uri.rindex("/") + 1 :] dbname_base = DATABASE_URI_BASE[DATABASE_URI_BASE.rindex("/") + 1 :] - dump_args = ["mysqldump", "-h", "db", "-u", "root", f"-p{mysql_root_password}", dbname_base] - load_args = ["mysql", "-h", "db", "-u", "root", f"-p{mysql_root_password}", dbname] - proc1 = subprocess.run(dump_args, capture_output=True) + dump_args = [ + "mariadb-dump", + "-h", + "db", + "-u", + "root", + f"-p{mysql_root_password}", + "--skip_ssl", + dbname_base, + ] + load_args = [ + "mariadb", + "-h", + "db", + "-u", + "root", + f"-p{mysql_root_password}", + "--skip_ssl", + dbname, + ] + proc1 = subprocess.run(dump_args, stdout=subprocess.PIPE) proc2 = subprocess.run(load_args, input=proc1.stdout, capture_output=True) + """ + # Dump the database + with open("/tmp/dump.sql", "wb") as f: + subprocess.run(dump_args, stdout=f) + + # This is necessary because the dump contains a line that is not supported by this mariadb client + # The line is M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 + with open("/tmp/dump.sql", "rb") as file: + lines = file.readlines() + with open("/tmp/dump.sql", "wb") as file: + for line in lines: + if b"NOTE_VERBOSITY" not in line: # Use binary strings to avoid encoding issues + file.write(line) + + # Load the database + with open("/tmp/dump.sql", "rb") as f: + subprocess.run(load_args, stdin=f) + """ + def demo_data(): from dds_web.utils import timestamp From 0d5c80397235126a1123102c5c3d5029ef75b273 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 11 Dec 2024 16:07:58 +0100 Subject: [PATCH 3/8] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 05c86da58..93701528e 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -464,3 +464,4 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) - Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) +- Tests failing due to mariadn client error ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) From 82745cbe71797fe35a81a5b515652a6189861ef3 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 11 Dec 2024 16:37:46 +0100 Subject: [PATCH 4/8] first solution to tests --- docker-compose.yml | 2 +- tests/conftest.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1df613bfa..7cda25b4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ version: "3.9" services: db: container_name: dds_database - image: mariadb:11.6.2 + image: mariadb:10.11.5 environment: - MYSQL_ROOT_PASSWORD=${DDS_MYSQL_ROOT_PASS} - MYSQL_USER=${DDS_MYSQL_USER} diff --git a/tests/conftest.py b/tests/conftest.py index e7fa1eab2..e6e343f01 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -124,10 +124,9 @@ def new_test_db(uri): dbname, ] - proc1 = subprocess.run(dump_args, stdout=subprocess.PIPE) - proc2 = subprocess.run(load_args, input=proc1.stdout, capture_output=True) + # proc1 = subprocess.run(dump_args, stdout=subprocess.PIPE) + # proc2 = subprocess.run(load_args, input=proc1.stdout, capture_output=True) - """ # Dump the database with open("/tmp/dump.sql", "wb") as f: subprocess.run(dump_args, stdout=f) @@ -144,7 +143,6 @@ def new_test_db(uri): # Load the database with open("/tmp/dump.sql", "rb") as f: subprocess.run(load_args, stdin=f) - """ def demo_data(): From 681b300a4c6e5ee49a231285887a203749d966f7 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Dec 2024 09:51:31 +0100 Subject: [PATCH 5/8] fix tests --- Dockerfiles/backend.Dockerfile | 7 ++++++- tests/conftest.py | 23 ++--------------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/Dockerfiles/backend.Dockerfile b/Dockerfiles/backend.Dockerfile index a98474321..6b0d639b0 100644 --- a/Dockerfiles/backend.Dockerfile +++ b/Dockerfiles/backend.Dockerfile @@ -49,7 +49,12 @@ ENV PYTHONPATH /code ################### FROM base as test RUN pip3 install -r /code/tests/requirements-test.txt -RUN apk add mariadb-client + +# The version of mariadb-client should match the version of the mariadb server +# Because of how alpine works, this is how to pin a version. However, it can break if this branch is removed from alpine +# https://superuser.com/questions/1055060/how-to-install-a-specific-package-version-in-alpine +# https://pkgs.alpinelinux.org/packages?name=mariadb-client&branch=v3.19&repo=&arch=x86_64&origin=&flagged=&maintainer= +RUN apk add mariadb-client=~10.11 --repository https://dl-cdn.alpinelinux.org/alpine/v3.19/main/ # Switch to the user USER $USERNAME diff --git a/tests/conftest.py b/tests/conftest.py index e6e343f01..608c40032 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,7 +110,6 @@ def new_test_db(uri): "-u", "root", f"-p{mysql_root_password}", - "--skip_ssl", dbname_base, ] load_args = [ @@ -120,29 +119,11 @@ def new_test_db(uri): "-u", "root", f"-p{mysql_root_password}", - "--skip_ssl", dbname, ] - # proc1 = subprocess.run(dump_args, stdout=subprocess.PIPE) - # proc2 = subprocess.run(load_args, input=proc1.stdout, capture_output=True) - - # Dump the database - with open("/tmp/dump.sql", "wb") as f: - subprocess.run(dump_args, stdout=f) - - # This is necessary because the dump contains a line that is not supported by this mariadb client - # The line is M!100616 SET @OLD_NOTE_VERBOSITY=@@NOTE_VERBOSITY, NOTE_VERBOSITY=0 - with open("/tmp/dump.sql", "rb") as file: - lines = file.readlines() - with open("/tmp/dump.sql", "wb") as file: - for line in lines: - if b"NOTE_VERBOSITY" not in line: # Use binary strings to avoid encoding issues - file.write(line) - - # Load the database - with open("/tmp/dump.sql", "rb") as f: - subprocess.run(load_args, stdin=f) + proc1 = subprocess.run(dump_args, stdout=subprocess.PIPE) + proc2 = subprocess.run(load_args, input=proc1.stdout, capture_output=True) def demo_data(): From e178477ecce289bc449ccc0102e9e9b258753ea3 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Dec 2024 09:52:27 +0100 Subject: [PATCH 6/8] sprintlog --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 93701528e..6290fbcc0 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -464,4 +464,4 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) - Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) -- Tests failing due to mariadn client error ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) +- update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) From 1208a0f39da70b123bcca6d36008be101942ab69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:06:05 +0100 Subject: [PATCH 7/8] Update SPRINTLOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 6290fbcc0..3fd3ce53f 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -464,4 +464,4 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) - Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) -- update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) +- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) From bf449a0c47605fa0f2d43f8b85a8bc3a3d2b837c Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:23:25 +0100 Subject: [PATCH 8/8] sprintlog - added entry and fixed missing ) --- SPRINTLOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 3fd3ce53f..75724dd06 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -463,5 +463,6 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) -- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) -- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) +- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575)) +- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581)) +- Update documentation regarding 'Upload' or 'Download' added to end of delivery directory name depending on command ([#1580](https://github.com/ScilifelabDataCentre/dds_web/pull/1580))