sprintlog and more test

ScilifelabDataCentre · Oct 31, 2023 · 9187e9f · 9187e9f
1 parent 46833f8
commit 9187e9f
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 36 deletions.
diff --git a/SPRINTLOG.md b/SPRINTLOG.md
@@ -314,3 +314,7 @@ _Nothing merged in CLI during this sprint_
 - Use full DDS name in MOTD email subject ([#1477](https://github.com/ScilifelabDataCentre/dds_web/pull/1477))
 - Add flag --verify-checksum to the comand in email template ([#1478])(https://github.com/ScilifelabDataCentre/dds_web/pull/1478)
 - Improved email layout; Highlighted information and commands when project is released ([#1479])(https://github.com/ScilifelabDataCentre/dds_web/pull/1479)
+
+# 2023-10-23 - 2023-11-3
+
+- Change the generate usage command to monthly instead of quartely, and add the command to send a usage report specifying the number of months ([#1476])[https://github.com/ScilifelabDataCentre/dds_web/pull/1476]
diff --git a/dds_web/commands.py b/dds_web/commands.py
@@ -1002,43 +1002,56 @@ def send_usage(months):
         total_usage = 0
 
         # Open new csv file
-        with csv_file_name.open(mode="w+", newline="") as file:
-            csv_writer = csv.writer(file)
-            csv_writer.writerow(
-                [
-                    "Project ID",
-                    "Project Title",
-                    "Project Created",
-                    "Time Collected",
-                    "Byte Hours",
-                ]
-            )
-
-            # Get usage rows connected to unit, that have been collected between X months ago and now
-            for usage_row, project_row in page_query(
-                db.session.query(models.Usage, models.Project)
-                .join(models.Project)
-                .filter(
-                    models.Project.responsible_unit == unit,
-                    models.Usage.time_collected.between(start, end),
-                )
-            ):
-                # Increase total unit usage
-                total_usage += usage_row.usage
-
-                # Save usage row info to csv file
+        try:
+            with csv_file_name.open(mode="w+", newline="") as file:
+                csv_writer = csv.writer(file)
                 csv_writer.writerow(
                     [
-                        project_row.public_id,
-                        project_row.title,
-                        project_row.date_created,
-                        usage_row.time_collected,
-                        usage_row.usage,
+                        "Project ID",
+                        "Project Title",
+                        "Project Created",
+                        "Time Collected",
+                        "Byte Hours",
                     ]
                 )
 
-            # Save total
-            csv_writer.writerow(["--", "--", "--", "--", total_usage])
+                # Get usage rows connected to unit, that have been collected between X months ago and now
+                for usage_row, project_row in page_query(
+                    db.session.query(models.Usage, models.Project)
+                    .join(models.Project)
+                    .filter(
+                        models.Project.responsible_unit == unit,
+                        models.Usage.time_collected.between(start, end),
+                    )
+                ):
+                    # Increase total unit usage
+                    total_usage += usage_row.usage
+
+                    # Save usage row info to csv file
+                    csv_writer.writerow(
+                        [
+                            project_row.public_id,
+                            project_row.title,
+                            project_row.date_created,
+                            usage_row.time_collected,
+                            usage_row.usage,
+                        ]
+                    )
+
+                # Save total
+                csv_writer.writerow(["--", "--", "--", "--", total_usage])
+        except Exception as e:
+            flask.current_app.logger.error(f"Error writing to CSV file: {e}")
+            # delete already generated csv files
+            [csv_file.unlink() for csv_file in csv_file_names]
+            # Send email about error
+            email_message: flask_mail.Message = flask_mail.Message(
+                subject=error_subject,
+                recipients=[email_recipient],
+                body=error_body,
+            )
+            send_email_with_retry(msg=email_message)
+            raise
 
     # Send email with the csv
     flask.current_app.logger.info("Sending email with the CSV.")
@@ -1054,6 +1067,9 @@ def send_usage(months):
             email_message.attach(filename=str(csv_file), content_type="text/csv", data=file.read())
     send_email_with_retry(msg=email_message)
 
+    # delete the csv after sending the email
+    [csv_file.unlink() for csv_file in csv_file_names]
+
 
 @click.command("stats")
 @flask.cli.with_appcontext

diff --git a/tests/test_commands.py b/tests/test_commands.py
@@ -1655,6 +1655,7 @@ def test_send_usage(client, cli_runner, capfd: LogCaptureFixture):
     months_to_test = 3
 
     # Loop to populate usage table with fake entries across the months
+    usage_list = []
     for i in range(months_to_test + 1):
         time = now - relativedelta(months=i)
         usage_1 = Usage(
@@ -1672,8 +1673,10 @@ def test_send_usage(client, cli_runner, capfd: LogCaptureFixture):
             usage=100,
             time_collected=time,
         )
-        db.session.add_all([usage_1, usage_2, usage_3])
-        db.session.commit()
+        usage_list.extend([usage_1, usage_2, usage_3])
+
+    db.session.add_all(usage_list)
+    db.session.commit()
 
     # Run command
     with mail.record_messages() as outbox:
@@ -1704,5 +1707,41 @@ def test_send_usage(client, cli_runner, capfd: LogCaptureFixture):
         for attachment, file_name in zip(outbox[-1].attachments, [csv_1_name, csv_2_name]):
             assert attachment.filename == file_name
             assert attachment.content_type == "text/csv"
-    # TODO: test that the csv files are correct
-    # TODO: Test errors in the csv handling
+
+    # retrieve the csv and check that it is correct
+    csv_1 = outbox[-1].attachments[0].data
+    csv_2 = outbox[-1].attachments[1].data
+    assert "Project ID,Project Title,Project Created,Time Collected,Byte Hours" in csv_1
+    assert "--,--,--,--,600.0" in csv_1
+    assert "Project ID,Project Title,Project Created,Time Collected,Byte Hours" in csv_2
+    assert "--,--,--,--,300.0" in csv_2
+
+    import re
+
+    csv_1 = re.split(",|\n", csv_1)  # split by comma or newline
+    assert csv_1.count("public_project_id") == 3
+    assert csv_1.count("second_public_project_id") == 3
+    assert csv_1.count("unit2testing") == 0
+    assert csv_1.count("100.0") == 6
+
+    csv_2 = re.split(",|\n", csv_2)
+    assert csv_2.count("public_project_id") == 0
+    assert csv_2.count("second_public_project_id") == 0
+    assert csv_2.count("unit2testing") == 3
+    assert csv_2.count("100.0") == 3
+
+
+def test_send_usage_error_csv(client, cli_runner, capfd: LogCaptureFixture):
+    """Test that checks errors in the csv handling"""
+
+    with mail.record_messages() as outbox:
+        with patch("csv.writer") as mock_writing_file:
+            mock_writing_file.side_effect = IOError()
+            cli_runner.invoke(send_usage, ["--months", 3])
+
+        _, logs = capfd.readouterr()
+        assert "Error writing to CSV file:" in logs
+        # Verify error email
+        assert len(outbox) == 1
+        assert "[SEND-USAGE CRONJOB] <ERROR> Error in send-usage cronjob" in outbox[-1].subject
+        assert "There was an error in the cronjob 'send-usage'" in outbox[-1].body