Merge branch 'dev' into HMS-2195-Fix-cross-spawn-regular-expression-d…

…enial-of-service
ScilifelabDataCentre · Dec 13, 2024 · de7183d · de7183d
2 parents da6e455 + 00fd250
commit de7183d
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 22 deletions.
diff --git a/SPRINTLOG.md b/SPRINTLOG.md
@@ -463,6 +463,8 @@ _Nothing merged during this sprint_
 
 - Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571))
 - Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573))
-- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575)
+- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575))
+- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581))
+- Update documentation regarding 'Upload' or 'Download' added to end of delivery directory name depending on command ([#1580](https://github.com/ScilifelabDataCentre/dds_web/pull/1580))
+- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562))
 - Run npm audit fix to solve node cve's ([#1577](https://github.com/ScilifelabDataCentre/dds_web/pull/1577)
-- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581)
diff --git a/dds_web/commands.py b/dds_web/commands.py
@@ -1269,7 +1269,7 @@ def monitor_usage():
     import dds_web.utils
 
     # Email settings
-    recipient: str = flask.current_app.config.get("MAIL_DDS")
+    dds_contact: str = flask.current_app.config.get("MAIL_DDS")
     default_subject: str = "DDS: Usage quota warning!"
 
     # Run task
@@ -1305,15 +1305,17 @@ def monitor_usage():
         # Email if the unit is using more
         if perc_used_decimal > warn_after:
             # Email settings
+            unit_contact: str = unit.contact_email
             message: str = (
-                "A SciLifeLab Unit is approaching the allocated data quota.\n"
-                f"Affected unit: {unit.name}\n"
+                "Your unit is approaching the allocated data quota (see details below).\n\n"
+                f"NB! If you would like to increase or decrease the allocated quota ('Quota') or the level after which you receive this email ('Warning level'), the technical contact person for your unit must send a request to {dds_contact}.\n"
+                f"Unit name: {unit.name}\n"
                 f"{info_string}"
             )
             flask.current_app.logger.info(message)
             msg: flask_mail.Message = flask_mail.Message(
                 subject=default_subject,
-                recipients=[recipient],
+                recipients=[unit_contact, dds_contact],
                 body=message,
             )
             dds_web.utils.send_email_with_retry(msg=msg)
diff --git a/dds_web/database/models.py b/dds_web/database/models.py
@@ -187,7 +187,7 @@ class Unit(db.Model):
     public_id = db.Column(db.String(50), unique=True, nullable=False)
     name = db.Column(db.String(255), unique=True, nullable=False)
     external_display_name = db.Column(db.String(255), unique=False, nullable=False)
-    contact_email = db.Column(db.String(255), unique=False, nullable=True)
+    contact_email = db.Column(db.String(255), unique=False, nullable=False)
     internal_ref = db.Column(db.String(50), unique=True, nullable=False)
 
     # Safespring storage

diff --git a/doc/technical-overview.md b/doc/technical-overview.md
@@ -177,7 +177,7 @@ production instance, and should be collected from the agreements or additional c
 | Days in Available                                 | The number of days during which the data will be available for download by the Researchers. The countdown starts when the project is released. There is no time limit when the project is In Progress and the project has not been released. For more information on the project statuses and what actions can be performed during them, see the appendix ([Project Statuses](#b-project-statuses)). After Days in Available (DiA) number of days has passed, the project is automatically set as Expired. |
 | Days in Expired                                   | The number of days (after being available) during which the data is still stored but not available for download. During this time, the project can be renewed, leading to the project being available again and therefore allowing for downloads again. When the Days in Expired (DiE) number of days has passed, the project is automatically archived by the system.                                                                                                                                     |
 | Quota                                             | The amount of storage space made available for a particular unit. This information should be included in the service agreement. Another value cannot be chosen by the Data Centre or by the unit.                                                                                                                                                                                                                                                                                                          |
-| Warning level                                     | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to [[email protected]](mailto:[email protected]). In the event of an alert, the Data Centre contacts the unit to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project.                                                   |
+| Warning level                                     | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to the contact email within the unit, as well as a to [[email protected]](mailto:[email protected]). In the event of an alert, the unit affected should contact the Data Centre to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project.                                                   |
 : Parameters used to configure a Unit within DDS 
 
 The unit also must provide the email addresses of at least two (but preferably three or more)
@@ -453,8 +453,8 @@ Personnel have the permissions to upload data.
 When starting an upload, a directory (“staging directory”) is created by the executing command. The
 default location of the staging directory is the current working directory, however the user can specify
 an existing directory in which the staging directory should be placed. Independent of the location
-(specified or default), the staging directory is named _DataDelivery*<timestamp>*<project_id>_,
-where <timestamp> is the date and time when the upload was started, and <project_id> is the ID of
+(specified or default), the staging directory is named _DataDelivery\_\<timestamp\>\_\<project_id\>\_upload_,
+where \<timestamp\> is the date and time when the upload was started, and \<project_id\> is the ID of
 the project the user is attempting to upload data to. If there is no data to upload, this directory is
 deleted immediately. If not, the staging directory will contain three subdirectories:
 
@@ -533,7 +533,7 @@ When downloading data, the Researchers can either choose to download specific fi
 folder(s), or the entire project contents.
 
 As with the upload ([Uploading data](#uploading-data)), a staging directory is created when downloading the data. This
-directory is placed by default in the current working directory, and is named DataDelivery*<timestamp>*<project_id>.
+directory is placed by default in the current working directory, and is named _DataDelivery\_\<timestamp\>\_\<project_id\>\_download_.
 However unlike the upload command, downloading allows the user to choose the name of the directory - specifying a
 destination. The destination cannot be an existing directory[^22] - it must be a new directory. Since a new destination is required with every
 download, downloading the same file(s) multiple times is possible and is only limited by the amount

diff --git a/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py b/migrations/versions/0cd0a3b251e0_unit_contact_email_non_nullable.py
@@ -0,0 +1,45 @@
+"""unit_contact_email_non_nullable
+
+Revision ID: 0cd0a3b251e0
+Revises: 3d610b382383
+Create Date: 2024-12-03 10:51:34.143028
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "0cd0a3b251e0"
+down_revision = "3d610b382383"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    unit_table = sa.sql.table(
+        "units", sa.sql.column("contact_email", mysql.TINYINT(display_width=1))
+    )
+    op.execute(
+        unit_table.update()
+        .where(unit_table.c.contact_email == None)
+        .values(contact_email="[email protected]")
+    )
+    op.alter_column(
+        "units",
+        "contact_email",
+        existing_type=mysql.VARCHAR(length=255),
+        nullable=False,
+        server_default="[email protected]",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "units", "contact_email", existing_type=mysql.VARCHAR(length=255), nullable=True
+    )
+    # ### end Alembic commands ###
diff --git a/tests/test_commands.py b/tests/test_commands.py
@@ -1432,19 +1432,21 @@ def test_monitor_usage_warning_sent(client, cli_runner, capfd: LogCaptureFixture
     # Mock the size property of the Unit table
     with patch("dds_web.database.models.Unit.size", new_callable=PropertyMock) as mock_size:
         mock_size.return_value = 0.9 * quota_in_test
-        # Mock emails - only check if function call
-        with patch.object(flask_mail.Mail, "send") as mock_mail_send:
+
+        with mail.record_messages() as outbox:
             # Run command
             _: click.testing.Result = cli_runner.invoke(monitor_usage)
-            # Verify no email has been sent and stoud contains logging info
-            assert mock_mail_send.call_count == 2  # 2 because client and cli_runner both run
-
-    _, err = capfd.readouterr()
-    for unit in models.Unit.query.all():
-        assert (
-            f"A SciLifeLab Unit is approaching the allocated data quota.\nAffected unit: {unit.name}\n"
-            in err
-        )
+            # capture output
+            _, err = capfd.readouterr()
+
+            i = 0
+            for unit in models.Unit.query.all():
+                # Verify email has been sent to the correct recipient
+                assert outbox[i].recipients[0] == unit.contact_email
+                assert outbox[i].recipients[1] == "[email protected]"
+                assert "Your unit is approaching the allocated data quota" in err
+                assert f"Unit name: {unit.name}" in err
+                i += 1
 
 
 # set_available_to_expired