From 7b97d3bb13623f573fae6bec04426a2f730e13fc Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 7 Jun 2024 19:37:21 +0400 Subject: [PATCH] Release 34.6.1 (#1260) * Remove print statements from migration files. Signed-off-by: tdruez * Display full traceback on error in execute command Signed-off-by: tdruez * Refactor the ``get_env_from_config_file`` to support empty config file Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 8 ++++++ scancodeio/__init__.py | 2 +- scanpipe/management/commands/__init__.py | 8 ++++-- .../0031_scancode_toolkit_v32_data_updates.py | 22 +++++++++------ .../0055_discoveredpackage_datafile_paths.py | 8 ++++-- scanpipe/models.py | 28 +++++++++++++------ setup.cfg | 4 +-- 7 files changed, 54 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c7aa1421d..4ab418c1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Changelog ========= +v34.6.1 (2024-06-07) +-------------------- + +- Remove print statements from migration files. +- Display full traceback on error in the ``execute`` management command. +- Log the Project message creation. +- Refactor the ``get_env_from_config_file`` to support empty config file. + v34.6.0 (2024-06-07) -------------------- diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 8051141a2..3f859ce75 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "34.6.0" +VERSION = "34.6.1" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 56b17e883..6187d8e1f 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -22,6 +22,7 @@ import shutil import sys +import traceback from pathlib import Path from django.apps import apps @@ -405,9 +406,10 @@ def execute_project(project, run_async=False, command=None): # noqa: C901 except KeyboardInterrupt: run.set_task_stopped() raise CommandError("Pipeline execution stopped.") - except Exception as e: - run.set_task_ended(exitcode=1, output=str(e)) - raise CommandError(e) + except Exception: + traceback_str = traceback.format_exc() + run.set_task_ended(exitcode=1, output=traceback_str) + raise CommandError(traceback_str) run.refresh_from_db() diff --git a/scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py b/scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py index c6a50cb9b..92a0439a4 100644 --- a/scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py +++ b/scanpipe/migrations/0031_scancode_toolkit_v32_data_updates.py @@ -1,9 +1,13 @@ # Generated by Django 4.2 on 2023-05-05 10:11 +import logging + from django.db import migrations from django.db.models import Q from django.conf import settings +logger = logging.getLogger("django") + def compute_package_declared_license_expression_spdx(apps, schema_editor): """ @@ -21,7 +25,7 @@ def compute_package_declared_license_expression_spdx(apps, schema_editor): ).only("declared_license_expression") object_count = queryset.count() - print(f"\nCompute declared_license_expression_spdx for {object_count:,} packages.") + logger.info(f"\nCompute declared_license_expression_spdx for {object_count:,} packages.") chunk_size = 2000 iterator = queryset.iterator(chunk_size=chunk_size) @@ -33,9 +37,9 @@ def compute_package_declared_license_expression_spdx(apps, schema_editor): unsaved_objects.append(package) if not (index % chunk_size) and unsaved_objects: - print(f" {index:,} / {object_count:,} computed") + logger.info(f" {index:,} / {object_count:,} computed") - print("Updating DB objects...") + logger.info("Updating DB objects...") DiscoveredPackage.objects.bulk_update( objs=unsaved_objects, fields=["declared_license_expression_spdx"], @@ -61,7 +65,7 @@ def compute_resource_detected_license_expression(apps, schema_editor): ) object_count = queryset.count() - print(f"\nCompute detected_license_expression for {object_count:,} resources.") + logger.info(f"\nCompute detected_license_expression for {object_count:,} resources.") chunk_size = 2000 iterator = queryset.iterator(chunk_size=chunk_size) @@ -88,9 +92,9 @@ def compute_resource_detected_license_expression(apps, schema_editor): unsaved_objects.append(resource) if not (index % chunk_size) and unsaved_objects: - print(f" {index:,} / {object_count:,} computed") + logger.info(f" {index:,} / {object_count:,} computed") - print("Updating DB objects...") + logger.info("Updating DB objects...") CodebaseResource.objects.bulk_update( objs=unsaved_objects, fields=[ @@ -164,7 +168,7 @@ def compute_resource_license_detections(apps, schema_editor): queryset = CodebaseResource.objects.filter(~Q(licenses=[])).only("licenses") object_count = queryset.count() - print(f"\nCompute license_detections for {object_count:,} resources.") + logger.info(f"\nCompute license_detections for {object_count:,} resources.") chunk_size = 2000 iterator = queryset.iterator(chunk_size=chunk_size) @@ -176,9 +180,9 @@ def compute_resource_license_detections(apps, schema_editor): unsaved_objects.append(resource) if not (index % chunk_size): - print(f" {index:,} / {object_count:,} computed") + logger.info(f" {index:,} / {object_count:,} computed") - print("Updating DB objects...") + logger.info("Updating DB objects...") # Keeping the batch_size small as the `license_detections` content is often large, # and it may raise `django.db.utils.OperationalError: out of memory` CodebaseResource.objects.bulk_update( diff --git a/scanpipe/migrations/0055_discoveredpackage_datafile_paths.py b/scanpipe/migrations/0055_discoveredpackage_datafile_paths.py index 6f43b2a33..e2fe610e3 100644 --- a/scanpipe/migrations/0055_discoveredpackage_datafile_paths.py +++ b/scanpipe/migrations/0055_discoveredpackage_datafile_paths.py @@ -1,8 +1,12 @@ # Generated by Django 5.0.2 on 2024-03-01 16:09 +import logging + from django.db import migrations, models from django.db.models import Q +logger = logging.getLogger("django") + def update_package_datasource_ids(apps, schema_editor): """ @@ -14,7 +18,7 @@ def update_package_datasource_ids(apps, schema_editor): object_count = queryset.count() if object_count: - print(f"\nCompute datasource_ids for {object_count:,} packages.") + logger.info(f"\nCompute datasource_ids for {object_count:,} packages.") chunk_size = 2000 iterator = queryset.iterator(chunk_size=chunk_size) @@ -26,7 +30,7 @@ def update_package_datasource_ids(apps, schema_editor): unsaved_objects.append(package) if not (index % chunk_size) and unsaved_objects: - print(f" {index:,} / {object_count:,} computed") + logger.info(f" {index:,} / {object_count:,} computed") DiscoveredPackage.objects.bulk_update( objs=unsaved_objects, diff --git a/scanpipe/models.py b/scanpipe/models.py index 33fceb127..474b71a32 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -804,6 +804,21 @@ def get_enabled_settings(self): if value not in EMPTY_VALUES } + def get_env_from_config_file(self): + """Return ``env`` dict loaded from the ``scancode-config.yml`` config file.""" + config_file = self.get_input_config_file() + if not config_file: + return + + logger.info(f"Loading env from {config_file.relative_to(self.work_path)}") + try: + return saneyaml.load(config_file.read_text()) + except (saneyaml.YAMLError, Exception): + self.add_error( + f'Failed to load configuration from "{config_file}". ' + f"The file format is invalid." + ) + def get_env(self, field_name=None): """ Return the project environment loaded from the ``scancode-config.yml`` config @@ -814,15 +829,8 @@ def get_env(self, field_name=None): env = {} # 1. Load settings from config file when available. - if config_file := self.get_input_config_file(): - logger.info(f"Loading env from {config_file.relative_to(self.work_path)}") - try: - env = saneyaml.load(config_file.read_text()) - except saneyaml.YAMLError: - self.add_error( - f'Failed to load configuration from "{config_file}". ' - f"The file format is invalid." - ) + if env_from_config_file := self.get_env_from_config_file(): + env = env_from_config_file # 2. Update with defined values from the Project ``settings`` field. env.update(self.get_enabled_settings()) @@ -1173,6 +1181,8 @@ def add_message( A ``resource`` can be provided to keep track of the codebase resource that was analyzed when the error occurred. """ + logger.info(f"[{severity}] {description}") + if inspect.isclass(model): model = model.__name__ diff --git a/setup.cfg b/setup.cfg index e881f0b3e..c6f7020b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancodeio -version = 34.6.0 +version = 34.6.1 license = Apache-2.0 description = Automate software composition analysis pipelines long_description = file:README.rst @@ -170,7 +170,7 @@ ignore = D1,D203,D205,D212,D400,D415 [bumpver] version_pattern = "MAJOR.MINOR.PATCH" -current_version = "34.6.0" +current_version = "34.6.1" [bumpver:file_patterns] setup.cfg =