From 0b9992e7345af2e8a7061b6974a0873f142409be Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 12:32:50 +0200 Subject: [PATCH 01/33] change recipient in warning level --- dds_web/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index a290cd7cc..9c023d77a 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1247,7 +1247,7 @@ def monitor_usage(): import dds_web.utils # Email settings - recipient: str = flask.current_app.config.get("MAIL_DDS") + # recipient: str = flask.current_app.config.get("MAIL_DDS") default_subject: str = "DDS: Usage quota warning!" # Run task @@ -1282,6 +1282,7 @@ def monitor_usage(): # Email if the unit is using more if perc_used_decimal > warn_after: + recipient: str = unit.email # Email settings message: str = ( "A SciLifeLab Unit is approaching the allocated data quota.\n" From fb39bd9a3ee906b30f89f4e2fad74787173014f2 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 12:53:48 +0200 Subject: [PATCH 02/33] updated technical overview --- doc/technical-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/technical-overview.md b/doc/technical-overview.md index d37040fb7..d2693e2f4 100644 --- a/doc/technical-overview.md +++ b/doc/technical-overview.md @@ -177,7 +177,7 @@ production instance, and should be collected from the agreements or additional c | Days in Available | The number of days during which the data will be available for download by the Researchers. The countdown starts when the project is released. There is no time limit when the project is In Progress and the project has not been released. For more information on the project statuses and what actions can be performed during them, see the appendix ([Project Statuses](#b-project-statuses)). After Days in Available (DiA) number of days has passed, the project is automatically set as Expired. | | Days in Expired | The number of days (after being available) during which the data is still stored but not available for download. During this time, the project can be renewed, leading to the project being available again and therefore allowing for downloads again. When the Days in Expired (DiE) number of days has passed, the project is automatically archived by the system. | | Quota | The amount of storage space made available for a particular unit. This information should be included in the service agreement. Another value cannot be chosen by the Data Centre or by the unit. | -| Warning level | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to [delivery@scilifelab.se](mailto:delivery@scilifelab.se). In the event of an alert, the Data Centre contacts the unit to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project. | +| Warning level | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to the contact email within the unit. In the event of an alert, the unit affected should contact the Data Centre to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project. | : Parameters used to configure a Unit within DDS The unit also must provide the email addresses of at least two (but preferably three or more) From 6ad37e91c9cd5296970a00eb5ec3abb83fd24fb7 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 12:55:17 +0200 Subject: [PATCH 03/33] correct email --- dds_web/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 9c023d77a..8499f52fe 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1282,7 +1282,7 @@ def monitor_usage(): # Email if the unit is using more if perc_used_decimal > warn_after: - recipient: str = unit.email + recipient: str = unit.contact_email # Email settings message: str = ( "A SciLifeLab Unit is approaching the allocated data quota.\n" From 34a964edca76c7db0cd2dec4fd4a2881322c8326 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 15:46:15 +0200 Subject: [PATCH 04/33] modify test for warning to check correct recipient --- tests/test_commands.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1450c7c8a..d3bfbd07d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1432,19 +1432,22 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture # Mock the size property of the Unit table with patch("dds_web.database.models.Unit.size", new_callable=PropertyMock) as mock_size: mock_size.return_value = 0.9 * quota_in_test - # Mock emails - only check if function call - with patch.object(flask_mail.Mail, "send") as mock_mail_send: + + with mail.record_messages() as outbox: # Run command _: click.testing.Result = cli_runner.invoke(monitor_usage) - # Verify no email has been sent and stoud contains logging info - assert mock_mail_send.call_count == 2 # 2 because client and cli_runner both run + # capture output + _, err = capfd.readouterr() - _, err = capfd.readouterr() - for unit in models.Unit.query.all(): - assert ( - f"A SciLifeLab Unit is approaching the allocated data quota.\nAffected unit: {unit.name}\n" - in err - ) + i = 0 + for unit in models.Unit.query.all(): + # Verify email has been sent to the correct recipient + assert outbox[i].recipients[0] == unit.contact_email + assert ( + f"A SciLifeLab Unit is approaching the allocated data quota.\nAffected unit: {unit.name}\n" + in err + ) + i += 1 # set_available_to_expired From cbac005c966f78fea557b7181e951541f91dd6c8 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 15:46:30 +0200 Subject: [PATCH 05/33] send email to correct recipient --- dds_web/commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 8499f52fe..aa028e6d2 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1247,7 +1247,6 @@ def monitor_usage(): import dds_web.utils # Email settings - # recipient: str = flask.current_app.config.get("MAIL_DDS") default_subject: str = "DDS: Usage quota warning!" # Run task @@ -1282,8 +1281,8 @@ def monitor_usage(): # Email if the unit is using more if perc_used_decimal > warn_after: - recipient: str = unit.contact_email # Email settings + recipient: str = unit.contact_email message: str = ( "A SciLifeLab Unit is approaching the allocated data quota.\n" f"Affected unit: {unit.name}\n" From 2af8de16aacfe532f407023a8e3be9ee17112dc3 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 15:47:54 +0200 Subject: [PATCH 06/33] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c955ccfc6..d21593819 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,3 +440,4 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) +- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From 18b1a1a597199b77fe87e5d90d2b893c7d36ea7e Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 11:36:09 +0200 Subject: [PATCH 07/33] updated sprintlog --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c9e87bb3b..010b3e954 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,6 +440,6 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) -- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) - Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) - Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) +- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From 0422e341e030d4e8124fc645294fbdf6151e3136 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 18 Oct 2024 11:25:57 +0200 Subject: [PATCH 08/33] modified monitor usage --- dds_web/commands.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index c40b0fdb5..95a33fe73 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1269,6 +1269,7 @@ def monitor_usage(): import dds_web.utils # Email settings + dds_contact: str = flask.current_app.config.get("MAIL_DDS") default_subject: str = "DDS: Usage quota warning!" # Run task @@ -1304,16 +1305,16 @@ def monitor_usage(): # Email if the unit is using more if perc_used_decimal > warn_after: # Email settings - recipient: str = unit.contact_email + unit_contact: str = unit.contact_email message: str = ( - "A SciLifeLab Unit is approaching the allocated data quota.\n" - f"Affected unit: {unit.name}\n" + "Your unit is approaching the allocated data quota.\n" + f"Unit name: {unit.name}\n" f"{info_string}" ) flask.current_app.logger.info(message) msg: flask_mail.Message = flask_mail.Message( subject=default_subject, - recipients=[recipient], + recipients=[unit_contact,dds_contact], body=message, ) dds_web.utils.send_email_with_retry(msg=msg) From 3d51dea2b048fbc7e9a82f789db4439d007bf756 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 18 Oct 2024 11:29:09 +0200 Subject: [PATCH 09/33] =?UTF-8?q?adapt=20tests=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 366cd9669..1298f01b1 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1444,7 +1444,7 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture # Verify email has been sent to the correct recipient assert outbox[i].recipients[0] == unit.contact_email assert ( - f"A SciLifeLab Unit is approaching the allocated data quota.\nAffected unit: {unit.name}\n" + f"Your unit is approaching the allocated data quota.\nUnit name: {unit.name}\n" in err ) i += 1 From 488fbcab1673a9b242e74a3dff0461c3ec1d81a0 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 18 Oct 2024 11:50:08 +0200 Subject: [PATCH 10/33] black linter --- dds_web/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 95a33fe73..c247e832f 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1314,7 +1314,7 @@ def monitor_usage(): flask.current_app.logger.info(message) msg: flask_mail.Message = flask_mail.Message( subject=default_subject, - recipients=[unit_contact,dds_contact], + recipients=[unit_contact, dds_contact], body=message, ) dds_web.utils.send_email_with_retry(msg=msg) From 0c14f3c7eb5595aa08ca21ef5b6b6a71f5e16283 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 22 Oct 2024 15:34:10 +0200 Subject: [PATCH 11/33] check for the second recipient in test --- tests/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1298f01b1..040fe45d2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1443,6 +1443,7 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture for unit in models.Unit.query.all(): # Verify email has been sent to the correct recipient assert outbox[i].recipients[0] == unit.contact_email + assert outbox[i].recipients[1] == "support@example.com" assert ( f"Your unit is approaching the allocated data quota.\nUnit name: {unit.name}\n" in err From ae0a96dc5a9c4c4ce406504ff5d11e3f71861386 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 22 Oct 2024 15:39:49 +0200 Subject: [PATCH 12/33] fix emails --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 040fe45d2..2e9b8053b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1443,7 +1443,7 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture for unit in models.Unit.query.all(): # Verify email has been sent to the correct recipient assert outbox[i].recipients[0] == unit.contact_email - assert outbox[i].recipients[1] == "support@example.com" + assert outbox[i].recipients[1] == "delivery@scilifelab.se" assert ( f"Your unit is approaching the allocated data quota.\nUnit name: {unit.name}\n" in err From 805273e1abb4d7fa7e6bfebfca5f04103d222e81 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 22 Oct 2024 15:41:13 +0200 Subject: [PATCH 13/33] spritnlog --- SPRINTLOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 1b6ffee38..c6e7e35f3 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -445,4 +445,3 @@ _Nothing merged during this sprint_ - Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) - Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564)) - New version & changelog([#1565](https://github.com/ScilifelabDataCentre/dds_web/pull/1565)) - From 5ec77b2858ce9bdcc1eb2b79af2fdbbf0a2ee005 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 23 Oct 2024 11:44:33 +0200 Subject: [PATCH 14/33] move entry to current sprint --- SPRINTLOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c6e7e35f3..571c5959e 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -442,6 +442,9 @@ _Nothing merged during this sprint_ - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) - Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) - Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) -- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) - Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564)) - New version & changelog([#1565](https://github.com/ScilifelabDataCentre/dds_web/pull/1565)) + +# 2024-10-23 - 2024-11-01 + +- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From d1081edb82ba3c9b8464f0d4f80f9bf0d0022413 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 23 Oct 2024 11:47:41 +0200 Subject: [PATCH 15/33] update technical overview --- doc/technical-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/technical-overview.md b/doc/technical-overview.md index d2693e2f4..cf2e5004a 100644 --- a/doc/technical-overview.md +++ b/doc/technical-overview.md @@ -177,7 +177,7 @@ production instance, and should be collected from the agreements or additional c | Days in Available | The number of days during which the data will be available for download by the Researchers. The countdown starts when the project is released. There is no time limit when the project is In Progress and the project has not been released. For more information on the project statuses and what actions can be performed during them, see the appendix ([Project Statuses](#b-project-statuses)). After Days in Available (DiA) number of days has passed, the project is automatically set as Expired. | | Days in Expired | The number of days (after being available) during which the data is still stored but not available for download. During this time, the project can be renewed, leading to the project being available again and therefore allowing for downloads again. When the Days in Expired (DiE) number of days has passed, the project is automatically archived by the system. | | Quota | The amount of storage space made available for a particular unit. This information should be included in the service agreement. Another value cannot be chosen by the Data Centre or by the unit. | -| Warning level | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to the contact email within the unit. In the event of an alert, the unit affected should contact the Data Centre to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project. | +| Warning level | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to the contact email within the unit, as well as a to [delivery@scilifelab.se](mailto:delivery@scilifelab.se). In the event of an alert, the unit affected should contact the Data Centre to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project. | : Parameters used to configure a Unit within DDS The unit also must provide the email addresses of at least two (but preferably three or more) From b030727c635fb4baf32c09aab1d93ec4e0bb0a6f Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 24 Oct 2024 09:17:09 +0200 Subject: [PATCH 16/33] typo --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 571c5959e..b2fd1c951 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -447,4 +447,4 @@ _Nothing merged during this sprint_ # 2024-10-23 - 2024-11-01 -- Modify the monitor usage command to send warning to the affected unit instead of Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) +- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From f0e55bf320b1a0ca3287160f0ba40a53e7ff121e Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 1 Nov 2024 11:17:40 +0100 Subject: [PATCH 17/33] prettier --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 9912ec164..0235c719c 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -453,4 +453,4 @@ _Nothing merged during this sprint_ # 2024-11-04 - 2024-11-15 -- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) \ No newline at end of file +- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From bd2e7fdc0fb98841ba1922cd15d10f57726f6f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 28 Nov 2024 16:39:45 +0100 Subject: [PATCH 18/33] Update dds_web/commands.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- dds_web/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index c247e832f..ddca483d0 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1307,7 +1307,8 @@ def monitor_usage(): # Email settings unit_contact: str = unit.contact_email message: str = ( - "Your unit is approaching the allocated data quota.\n" + "Your unit is approaching the allocated data quota (see details below).\n\n" + f"NB! If you would like to increase or decrease the allocated quota ('Quota') or the level after which you receive this email ('Warning level'), the technical contact person for your unit must send a request to {dds_contact}.\n" f"Unit name: {unit.name}\n" f"{info_string}" ) From 2259ac4b2b25ed2acfa73817e76e7a9aeda61841 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 2 Dec 2024 13:29:14 +0100 Subject: [PATCH 19/33] migration --- dds_web/database/models.py | 2 +- ...8fde71e_unit_contact_email_non_nullable.py | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py diff --git a/dds_web/database/models.py b/dds_web/database/models.py index eb992339f..ff0ef8f6f 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -187,7 +187,7 @@ class Unit(db.Model): public_id = db.Column(db.String(50), unique=True, nullable=False) name = db.Column(db.String(255), unique=True, nullable=False) external_display_name = db.Column(db.String(255), unique=False, nullable=False) - contact_email = db.Column(db.String(255), unique=False, nullable=True) + contact_email = db.Column(db.String(255), unique=False, nullable=False) internal_ref = db.Column(db.String(50), unique=True, nullable=False) # Safespring storage diff --git a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py new file mode 100644 index 000000000..f544b652f --- /dev/null +++ b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py @@ -0,0 +1,33 @@ +"""unit_contact_email_non_nullable + +Revision ID: e02fe8fde71e +Revises: +Create Date: 2024-12-02 13:08:43.073488 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "e02fe8fde71e" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + table_name="units", + column_name="contact_email", + nullable=False, + ) + + +def downgrade(): + op.alter_column( + table_name="units", + column_name="contact_email", + nullable=False, + ) From 4b0a6ddf60bb309b39c16bebbc2c5cf7fe027df6 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 2 Dec 2024 13:34:25 +0100 Subject: [PATCH 20/33] fix tests --- tests/test_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e9b8053b..40855f1ef 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1444,10 +1444,8 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture # Verify email has been sent to the correct recipient assert outbox[i].recipients[0] == unit.contact_email assert outbox[i].recipients[1] == "delivery@scilifelab.se" - assert ( - f"Your unit is approaching the allocated data quota.\nUnit name: {unit.name}\n" - in err - ) + assert "Your unit is approaching the allocated data quota" in err + assert f"Unit name: {unit.name}" in err i += 1 From 17a11b404deff43da043176915218aea7f2f6525 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 3 Dec 2024 10:35:49 +0100 Subject: [PATCH 21/33] move sprintlog --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 880e7cf0a..706db7535 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -453,7 +453,6 @@ _Nothing merged during this sprint_ # 2024-11-04 - 2024-11-15 -- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) - Removed exception for invalid token to simplify logging and reduce unnecessary error entries ([#1572](https://github.com/ScilifelabDataCentre/dds_web/pull/1572)) # 2024-11-18 – 2024-11-29 @@ -465,3 +464,4 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) - Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) +- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From e3e60866cf6386810b1e9c4600a38b3ae744ab4a Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 3 Dec 2024 11:18:18 +0100 Subject: [PATCH 22/33] fix migration --- .../e02fe8fde71e_unit_contact_email_non_nullable.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py index f544b652f..378045502 100644 --- a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py +++ b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = "e02fe8fde71e" -down_revision = None +down_revision = "3d610b382383" branch_labels = None depends_on = None @@ -21,6 +21,7 @@ def upgrade(): op.alter_column( table_name="units", column_name="contact_email", + existing_type=sa.String(length=255), nullable=False, ) @@ -29,5 +30,6 @@ def downgrade(): op.alter_column( table_name="units", column_name="contact_email", - nullable=False, + existing_type=sa.String(length=255), + nullable=True, ) From c3c871b3bb6115b05058ab382b0a8323ab8758cb Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 3 Dec 2024 11:25:54 +0100 Subject: [PATCH 23/33] set default --- .../versions/e02fe8fde71e_unit_contact_email_non_nullable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py index 378045502..2b977369e 100644 --- a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py +++ b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py @@ -22,6 +22,7 @@ def upgrade(): table_name="units", column_name="contact_email", existing_type=sa.String(length=255), + server_default="delivery@scilifelab.se", nullable=False, ) From 557a693d3293fcd3173d1d8653f7e79e07ed28a3 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 3 Dec 2024 12:00:43 +0100 Subject: [PATCH 24/33] proper migration file --- ...3b251e0_unit_contact_email_non_nullable.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py new file mode 100644 index 000000000..956f69a11 --- /dev/null +++ b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py @@ -0,0 +1,33 @@ +"""unit_contact_email_non_nullable + +Revision ID: 0cd0a3b251e0 +Revises: 3d610b382383 +Create Date: 2024-12-03 10:51:34.143028 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '0cd0a3b251e0' +down_revision = '3d610b382383' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('units', 'contact_email', + existing_type=mysql.VARCHAR(length=255), + nullable=False, + server_default="delivery@scilifelab.se") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('units', 'contact_email', + existing_type=mysql.VARCHAR(length=255), + nullable=True) + # ### end Alembic commands ### From a99aaf0abb3ab35e1ea26531b0d17e1ecb5e14b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:02:15 +0100 Subject: [PATCH 25/33] Delete migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py --- ...8fde71e_unit_contact_email_non_nullable.py | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100644 migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py diff --git a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py b/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py deleted file mode 100644 index 2b977369e..000000000 --- a/migrations/versions/e02fe8fde71e_unit_contact_email_non_nullable.py +++ /dev/null @@ -1,36 +0,0 @@ -"""unit_contact_email_non_nullable - -Revision ID: e02fe8fde71e -Revises: -Create Date: 2024-12-02 13:08:43.073488 - -""" - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = "e02fe8fde71e" -down_revision = "3d610b382383" -branch_labels = None -depends_on = None - - -def upgrade(): - op.alter_column( - table_name="units", - column_name="contact_email", - existing_type=sa.String(length=255), - server_default="delivery@scilifelab.se", - nullable=False, - ) - - -def downgrade(): - op.alter_column( - table_name="units", - column_name="contact_email", - existing_type=sa.String(length=255), - nullable=True, - ) From d3897288281a58f3ed15685208d1510b9f56bba5 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 3 Dec 2024 12:02:47 +0100 Subject: [PATCH 26/33] black --- ...3b251e0_unit_contact_email_non_nullable.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py index 956f69a11..045b18d37 100644 --- a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py +++ b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py @@ -5,29 +5,33 @@ Create Date: 2024-12-03 10:51:34.143028 """ + from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. -revision = '0cd0a3b251e0' -down_revision = '3d610b382383' +revision = "0cd0a3b251e0" +down_revision = "3d610b382383" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('units', 'contact_email', - existing_type=mysql.VARCHAR(length=255), - nullable=False, - server_default="delivery@scilifelab.se") + op.alter_column( + "units", + "contact_email", + existing_type=mysql.VARCHAR(length=255), + nullable=False, + server_default="delivery@scilifelab.se", + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('units', 'contact_email', - existing_type=mysql.VARCHAR(length=255), - nullable=True) + op.alter_column( + "units", "contact_email", existing_type=mysql.VARCHAR(length=255), nullable=True + ) # ### end Alembic commands ### From 736cd47d53f1cfa6acf6fcda40fc483aca101d81 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 4 Dec 2024 02:56:41 +0100 Subject: [PATCH 27/33] added a proper readme --- migrations/README | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/migrations/README b/migrations/README index 0e0484415..d630a353d 100644 --- a/migrations/README +++ b/migrations/README @@ -1 +1,39 @@ -Single-database configuration for Flask. +# Database Migration Guide + +This document explains how to perform database migrations for this project. In DDS we use Alembic leveraged through flask-migrate. + +## Steps for Running Migrations + +### 1. Start the Containers + +Before making any model changes, ensure the containers are running locally. + +### 2. Modify the Database Models + +Edit the `models.py` file to reflect the changes you need. This might include: + +- Adding new tables or columns +- Modifying existing fields +- Deleting tables or columns + +### 3. Access the Backend Container + +Connect a terminal into the backend container. + +### 4. Run the Migration Command + +Inside the container, execute the migration command: + +```bash +flask db migrate -m "Your migration message" +``` + +### 5. Review the Generated Migration File + +A new migration script will be generated in the `migrations/versions` directory. **Review this file carefully** to ensure it correctly represents your intended changes. + +For reference on available operations and customization options, check the [Alembic Documentation](https://alembic.sqlalchemy.org/en/latest/ops.html). + +### 6. Commit the Migration File + +Finally, commit the migration script to version control. From abb19f6d9efe48cfc38a8e13890a18765865cc55 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 4 Dec 2024 12:24:19 +0100 Subject: [PATCH 28/33] undo readme --- migrations/README | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/migrations/README b/migrations/README index d630a353d..0e0484415 100644 --- a/migrations/README +++ b/migrations/README @@ -1,39 +1 @@ -# Database Migration Guide - -This document explains how to perform database migrations for this project. In DDS we use Alembic leveraged through flask-migrate. - -## Steps for Running Migrations - -### 1. Start the Containers - -Before making any model changes, ensure the containers are running locally. - -### 2. Modify the Database Models - -Edit the `models.py` file to reflect the changes you need. This might include: - -- Adding new tables or columns -- Modifying existing fields -- Deleting tables or columns - -### 3. Access the Backend Container - -Connect a terminal into the backend container. - -### 4. Run the Migration Command - -Inside the container, execute the migration command: - -```bash -flask db migrate -m "Your migration message" -``` - -### 5. Review the Generated Migration File - -A new migration script will be generated in the `migrations/versions` directory. **Review this file carefully** to ensure it correctly represents your intended changes. - -For reference on available operations and customization options, check the [Alembic Documentation](https://alembic.sqlalchemy.org/en/latest/ops.html). - -### 6. Commit the Migration File - -Finally, commit the migration script to version control. +Single-database configuration for Flask. From 0c94135d67b9627522295515af6287d502f539e2 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 5 Dec 2024 13:41:15 +0100 Subject: [PATCH 29/33] set default for null entries --- .../versions/0cd0a3b251e0_unit_contact_email_non_nullable.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py index 045b18d37..6002ffdfb 100644 --- a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py +++ b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py @@ -19,6 +19,9 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.execute( + "UPDATE units SET contact_email = 'delivery@scilifelab.se' WHERE contact_email IS NULL" + ) op.alter_column( "units", "contact_email", From 8357a8ac76103dbf80f93f53edd8581dc9c010e9 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Dec 2024 14:59:51 +0100 Subject: [PATCH 30/33] merge --- SPRINTLOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index fc2d49ec0..c41783b96 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -464,5 +464,6 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) - Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) -- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) - Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) +- Update documentation regarding 'Upload' or 'Download' added to end of delivery directory name depending on command ([#1580](https://github.com/ScilifelabDataCentre/dds_web/pull/1580)) +- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From 346424abd74563bfe5b95e44aafe91042d0698a1 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Dec 2024 15:01:10 +0100 Subject: [PATCH 31/33] merge --- SPRINTLOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c41783b96..fc1d90ec1 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -463,7 +463,7 @@ _Nothing merged during this sprint_ - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571)) - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573)) -- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575) -- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581) +- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575)) +- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581)) - Update documentation regarding 'Upload' or 'Download' added to end of delivery directory name depending on command ([#1580](https://github.com/ScilifelabDataCentre/dds_web/pull/1580)) - Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562)) From 13bc6dea0921d6cb177f5c665733853c0f78fa80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:22:45 +0100 Subject: [PATCH 32/33] Update migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- .../versions/0cd0a3b251e0_unit_contact_email_non_nullable.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py index 6002ffdfb..04b997f24 100644 --- a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py +++ b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py @@ -19,9 +19,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute( - "UPDATE units SET contact_email = 'delivery@scilifelab.se' WHERE contact_email IS NULL" - ) + unit_table = sa.sql.table("units", sa.sql.column("contact_email", mysql.TINYINT(display_width=1))) + op.execute(unit_table.update().where(unit_table.c.contact_email == None).values(contact_email="delivery@scilifelab.se")) op.alter_column( "units", "contact_email", From 260b2e9fca04a8f80dd52bd5474126725596508a Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 13 Dec 2024 09:26:42 +0100 Subject: [PATCH 33/33] black --- .../0cd0a3b251e0_unit_contact_email_non_nullable.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py index 04b997f24..4e12d8772 100644 --- a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py +++ b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py @@ -19,8 +19,14 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - unit_table = sa.sql.table("units", sa.sql.column("contact_email", mysql.TINYINT(display_width=1))) - op.execute(unit_table.update().where(unit_table.c.contact_email == None).values(contact_email="delivery@scilifelab.se")) + unit_table = sa.sql.table( + "units", sa.sql.column("contact_email", mysql.TINYINT(display_width=1)) + ) + op.execute( + unit_table.update() + .where(unit_table.c.contact_email == None) + .values(contact_email="delivery@scilifelab.se") + ) op.alter_column( "units", "contact_email",