From 54e63b61101c2593f7771408cb631a48dafe475a Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:54:59 +0400 Subject: [PATCH] Keep the InputSource objects when using reset on Projects #1536 (#1549) * Keep the InputSource objects when using ``reset`` on Projects #1536 Signed-off-by: tdruez * Remove connectivity requirement for a unit test Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 3 ++ scanpipe/models.py | 10 +++-- scanpipe/tests/__init__.py | 34 +++++++++++----- scanpipe/tests/pipes/test_output.py | 10 +---- scanpipe/tests/test_api.py | 10 +---- scanpipe/tests/test_models.py | 63 +++++++++++++++++------------ scanpipe/tests/test_pipelines.py | 10 +++-- 7 files changed, 81 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e930bb49..7a0c2b5f1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -47,6 +47,9 @@ v34.9.4 (unreleased) sheets with a dedicated VULNERABILITIES sheet. https://github.com/aboutcode-org/scancode.io/issues/1519 +- Keep the InputSource objects when using ``reset`` on Projects. + https://github.com/aboutcode-org/scancode.io/issues/1536 + - Add a ``report`` management command that allows to generate XLSX reports for multiple projects at once using labels and searching by project name. https://github.com/aboutcode-org/scancode.io/issues/1524 diff --git a/scanpipe/models.py b/scanpipe/models.py index e419a79e2..02d8a8e6d 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -640,14 +640,14 @@ def archive(self, remove_input=False, remove_codebase=False, remove_output=False self.is_archived = True self.save(update_fields=["is_archived"]) - def delete_related_objects(self): + def delete_related_objects(self, keep_input=False): """ Delete all related object instances using the private `_raw_delete` model API. This bypass the objects collection, cascade deletions, and signals. It results in a much faster objects deletion, but it needs to be applied in the correct models order as the cascading event will not be triggered. Note that this approach is used in Django's `fast_deletes` but the scanpipe - models are cannot be fast-deleted as they have cascades and relations. + models cannot be fast-deleted as they have cascades and relations. """ # Use default `delete()` on the DiscoveredPackage model, as the # `codebase_resources (ManyToManyField)` records need to collected and @@ -667,9 +667,11 @@ def delete_related_objects(self): self.discovereddependencies, self.codebaseresources, self.runs, - self.inputsources, ] + if not keep_input: + relationships.append(self.inputsources) + for qs in relationships: count = qs.all()._raw_delete(qs.db) deleted_counter[qs.model._meta.label] = count @@ -695,7 +697,7 @@ def reset(self, keep_input=True): """ self._raise_if_run_in_progress() - self.delete_related_objects() + self.delete_related_objects(keep_input=keep_input) work_directories = [ self.codebase_path, diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index fe2451cee..502412750 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -31,6 +31,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project +from scanpipe.models import ProjectMessage from scanpipe.tests.pipelines.do_nothing import DoNothing from scanpipe.tests.pipelines.download_inputs import DownloadInput from scanpipe.tests.pipelines.profile_step import ProfileStep @@ -47,12 +48,12 @@ mocked_now = mock.Mock(now=lambda: datetime(2010, 10, 10, 10, 10, 10)) -def make_project(name=None, **extra): +def make_project(name=None, **data): name = name or str(uuid.uuid4())[:8] - return Project.objects.create(name=name, **extra) + return Project.objects.create(name=name, **data) -def make_resource_file(project, path, **extra): +def make_resource_file(project, path, **data): return CodebaseResource.objects.create( project=project, path=path, @@ -61,30 +62,43 @@ def make_resource_file(project, path, **extra): type=CodebaseResource.Type.FILE, is_text=True, tag=path.split("/")[0], - **extra, + **data, ) -def make_resource_directory(project, path, **extra): +def make_resource_directory(project, path, **data): return CodebaseResource.objects.create( project=project, path=path, name=path.split("/")[-1], type=CodebaseResource.Type.DIRECTORY, tag=path.split("/")[0], - **extra, + **data, ) -def make_package(project, package_url, **extra): - package = DiscoveredPackage(project=project, **extra) +def make_package(project, package_url, **data): + package = DiscoveredPackage(project=project, **data) package.set_package_url(package_url) package.save() return package -def make_dependency(project, **extra): - return DiscoveredDependency.objects.create(project=project, **extra) +def make_dependency(project, **data): + return DiscoveredDependency.objects.create(project=project, **data) + + +def make_message(project, **data): + if "model" not in data: + data["model"] = str(uuid.uuid4())[:8] + + if "severity" not in data: + data["severity"] = ProjectMessage.Severity.ERROR + + return ProjectMessage.objects.create( + project=project, + **data, + ) resource_data1 = { diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 88d20982b..25104710f 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -42,11 +42,11 @@ from scanpipe import pipes from scanpipe.models import CodebaseResource from scanpipe.models import Project -from scanpipe.models import ProjectMessage from scanpipe.pipes import flag from scanpipe.pipes import output from scanpipe.tests import FIXTURES_REGEN from scanpipe.tests import make_dependency +from scanpipe.tests import make_message from scanpipe.tests import make_package from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now @@ -206,13 +206,7 @@ def test_scanpipe_pipes_outputs_to_xlsx(self): call_command("loaddata", fixtures, **{"verbosity": 0}) project = Project.objects.get(name="asgiref") - ProjectMessage.objects.create( - project=project, - severity=ProjectMessage.Severity.ERROR, - description="Error", - model="Model", - details={}, - ) + make_message(project, description="Error") make_resource_file( project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW ) diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index d91087946..5969b09da 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -54,6 +54,7 @@ from scanpipe.pipes.input import copy_input from scanpipe.pipes.output import JSONResultsGenerator from scanpipe.tests import dependency_data1 +from scanpipe.tests import make_message from scanpipe.tests import make_package from scanpipe.tests import make_project from scanpipe.tests import make_resource_file @@ -795,13 +796,7 @@ def test_scanpipe_api_project_action_relations_filterset(self): def test_scanpipe_api_project_action_messages(self): url = reverse("project-messages", args=[self.project1.uuid]) - ProjectMessage.objects.create( - project=self.project1, - severity=ProjectMessage.Severity.ERROR, - description="Error", - model="ModelName", - details={}, - ) + make_message(self.project1, description="Error") response = self.csrf_client.get(url) self.assertEqual(1, response.data["count"]) @@ -812,7 +807,6 @@ def test_scanpipe_api_project_action_messages(self): message = response.data["results"][0] self.assertEqual("error", message["severity"]) self.assertEqual("Error", message["description"]) - self.assertEqual("ModelName", message["model"]) self.assertEqual({}, message["details"]) def test_scanpipe_api_project_action_file_content(self): diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 71064fcdf..267f8e98f 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -76,7 +76,9 @@ from scanpipe.tests import dependency_data2 from scanpipe.tests import license_policies_index from scanpipe.tests import make_dependency +from scanpipe.tests import make_message from scanpipe.tests import make_package +from scanpipe.tests import make_project from scanpipe.tests import make_resource_directory from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now @@ -93,7 +95,7 @@ class ScanPipeModelsTest(TestCase): fixtures = [data / "asgiref" / "asgiref-3.3.0_fixtures.json"] def setUp(self): - self.project1 = Project.objects.create(name="Analysis") + self.project1 = make_project("Analysis") self.project_asgiref = Project.objects.get(name="asgiref") def create_run(self, pipeline="pipeline", **kwargs): @@ -118,7 +120,7 @@ def test_scanpipe_project_model_work_directories(self): self.assertTrue(self.project1.tmp_path.exists()) def test_scanpipe_get_project_work_directory(self): - project = Project.objects.create(name="Name with spaces and @£$éæ") + project = make_project("Name with spaces and @£$éæ") expected = f"/projects/name-with-spaces-and-e-{project.short_uuid}" self.assertTrue(get_project_work_directory(project).endswith(expected)) self.assertTrue(project.work_directory.endswith(expected)) @@ -191,7 +193,7 @@ def test_scanpipe_project_model_delete(self): self.assertTrue(work_path.exists()) uploaded_file = SimpleUploadedFile("file.ext", content=b"content") - self.project1.write_input_file(uploaded_file) + self.project1.add_upload(uploaded_file=uploaded_file, tag="tag1") self.project1.add_pipeline("analyze_docker_image") resource = CodebaseResource.objects.create(project=self.project1, path="path") package = DiscoveredPackage.objects.create(project=self.project1) @@ -209,11 +211,18 @@ def test_scanpipe_project_model_reset(self): self.assertTrue(work_path.exists()) uploaded_file = SimpleUploadedFile("file.ext", content=b"content") - self.project1.write_input_file(uploaded_file) + self.project1.add_upload(uploaded_file=uploaded_file, tag="tag1") self.project1.add_pipeline("analyze_docker_image") resource = CodebaseResource.objects.create(project=self.project1, path="path") package = DiscoveredPackage.objects.create(project=self.project1) resource.discovered_packages.add(package) + make_message(self.project1, description="Error") + + self.assertEqual(1, self.project1.projectmessages.count()) + self.assertEqual(1, self.project1.runs.count()) + self.assertEqual(1, self.project1.discoveredpackages.count()) + self.assertEqual(1, self.project1.codebaseresources.count()) + self.assertEqual(1, self.project1.inputsources.count()) self.project1.reset() @@ -223,6 +232,8 @@ def test_scanpipe_project_model_reset(self): self.assertEqual(0, self.project1.discoveredpackages.count()) self.assertEqual(0, self.project1.codebaseresources.count()) + # The InputSource objects are kept + self.assertEqual(1, self.project1.inputsources.count()) self.assertTrue(work_path.exists()) self.assertTrue(self.project1.input_path.exists()) self.assertEqual(["file.ext"], self.project1.input_root) @@ -604,7 +615,7 @@ def test_scanpipe_project_related_model_clone(self): target_url="http://domain.url" ) - new_project = Project.objects.create(name="New Project") + new_project = make_project("New Project") subscription1.clone(to_project=new_project) cloned_subscription = new_project.webhooksubscriptions.get() @@ -1884,7 +1895,7 @@ def test_scanpipe_discovered_package_queryset_vulnerable(self): self.assertIn(p2, DiscoveredPackage.objects.vulnerable()) def test_scanpipe_discovered_package_queryset_dependency_methods(self): - project = Project.objects.create(name="project") + project = make_project("project") a = make_package(project, "pkg:type/a") b = make_package(project, "pkg:type/b") c = make_package(project, "pkg:type/c") @@ -1978,7 +1989,7 @@ def test_scanpipe_codebase_resource_model_walk_method(self): self.assertEqual(sorted(expected_siblings), sorted(asgiref_resource_siblings)) def test_scanpipe_codebase_resource_model_walk_method_problematic_filenames(self): - project = Project.objects.create(name="walk_test_problematic_filenames") + project = make_project("walk_test_problematic_filenames") resource1 = CodebaseResource.objects.create( project=project, path="qt-everywhere-opensource-src-5.3.2/gnuwin32/bin" ) @@ -2331,7 +2342,7 @@ def test_scanpipe_discovered_dependency_model_update_from_data(self): self.assertEqual(new_data["scope"], dependency.scope) def test_scanpipe_discovered_dependency_model_many_to_many(self): - project = Project.objects.create(name="project") + project = make_project("project") a = make_package(project, "pkg:type/a") b = make_package(project, "pkg:type/b") @@ -2432,7 +2443,7 @@ def test_scanpipe_codebase_resource_queryset_has_directory_content_fingerprint( self.assertQuerySetEqual(expected, results, ordered=False) def test_scanpipe_codebase_resource_queryset_elfs(self): - project = Project.objects.create(name="Test") + project = make_project("Test") resource_starting_with_elf_and_executable_in_file_type = CodebaseResource( file_type="""ELF 32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", @@ -2510,7 +2521,7 @@ class ScanPipeModelsTransactionTest(TransactionTestCase): @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertEqual(0, project1.runs.count()) @@ -2529,13 +2540,13 @@ def test_scanpipe_project_model_add_pipeline(self, mock_execute_task): self.assertEqual(pipeline_class.get_summary(), run.description) mock_execute_task.assert_not_called() - project2 = Project.objects.create(name="Analysis 2") + project2 = make_project("Analysis 2") project2.add_pipeline(pipeline_name, execute_now=True) mock_execute_task.assert_called_once() @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline_run_can_start(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "inspect_packages" run1 = project1.add_pipeline(pipeline_name, execute_now=False) run2 = project1.add_pipeline(pipeline_name, execute_now=True) @@ -2547,7 +2558,7 @@ def test_scanpipe_project_model_add_pipeline_run_can_start(self, mock_execute_ta @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline_start_method(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "inspect_packages" run1 = project1.add_pipeline(pipeline_name, execute_now=False) run2 = project1.add_pipeline(pipeline_name, execute_now=False) @@ -2564,7 +2575,7 @@ def test_scanpipe_project_model_add_pipeline_start_method(self, mock_execute_tas mock_execute_task.assert_called_once() def test_scanpipe_project_model_add_pipeline_selected_groups(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "scan_codebase" run1 = project1.add_pipeline(pipeline_name, selected_groups=[]) @@ -2580,7 +2591,7 @@ def test_scanpipe_project_model_add_pipeline_selected_groups(self): project1.add_pipeline(pipeline_name, selected_groups={}) def test_scanpipe_project_model_add_info(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") message = project1.add_info(description="This is an info") self.assertEqual(message, ProjectMessage.objects.get()) self.assertEqual("", message.model) @@ -2590,7 +2601,7 @@ def test_scanpipe_project_model_add_info(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_add_warning(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") message = project1.add_warning(description="This is a warning") self.assertEqual(message, ProjectMessage.objects.get()) self.assertEqual("", message.model) @@ -2600,7 +2611,7 @@ def test_scanpipe_project_model_add_warning(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_add_error(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") details = { "name": "value", "release_date": datetime.fromisoformat("2008-02-01"), @@ -2618,7 +2629,7 @@ def test_scanpipe_project_model_add_error(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_update_extra_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertEqual({}, project1.extra_data) with self.assertRaises(ValueError): @@ -2647,7 +2658,7 @@ def test_scanpipe_project_model_update_extra_data(self): self.assertEqual(expected, project1.extra_data) def test_scanpipe_codebase_resource_model_add_error(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") codebase_resource = CodebaseResource.objects.create(project=project1, path="a") error = codebase_resource.add_error(Exception("Error message")) @@ -2659,7 +2670,7 @@ def test_scanpipe_codebase_resource_model_add_error(self): self.assertEqual(codebase_resource.path, error.details["resource_path"]) def test_scanpipe_codebase_resource_model_add_errors(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") codebase_resource = CodebaseResource.objects.create(project=project1) codebase_resource.add_error(Exception("Error1")) codebase_resource.add_error(Exception("Error2")) @@ -2667,7 +2678,7 @@ def test_scanpipe_codebase_resource_model_add_errors(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_project_error_model_save_non_valid_related_object(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") long_value = "value" * 1000 package = DiscoveredPackage.objects.create( @@ -2695,7 +2706,7 @@ def test_scanpipe_project_error_model_save_non_valid_related_object(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_package_model_create_from_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") package = DiscoveredPackage.create_from_data(project1, package_data1) self.assertEqual(project1, package.project) @@ -2737,7 +2748,7 @@ def test_scanpipe_discovered_package_model_create_from_data(self): self.assertEqual(project_message_count, ProjectMessage.objects.count()) def test_scanpipe_discovered_package_model_create_from_data_missing_type(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") incomplete_data = dict(package_data1) incomplete_data["type"] = "" @@ -2749,7 +2760,7 @@ def test_scanpipe_discovered_package_model_create_from_data_missing_type(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_dependency_model_create_from_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") DiscoveredPackage.create_from_data(project1, package_data1) CodebaseResource.objects.create( @@ -2797,7 +2808,7 @@ def test_scanpipe_discovered_dependency_model_create_from_data(self): self.assertEqual("", message.traceback) def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertTrue(package_data1["package_uid"]) package = DiscoveredPackage.create_from_data(project1, package_data1) @@ -2817,7 +2828,7 @@ def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_codebase_resource_create_and_add_package_warnings(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") resource = CodebaseResource.objects.create(project=project1, path="p") package_count = DiscoveredPackage.objects.count() diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index bca1ec48f..5c864a8f7 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1271,7 +1271,10 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_empty_manifest(self) expected = "No packages could be resolved" self.assertIn(expected, message.description) - def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): + @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") + def test_scanpipe_resolve_dependencies_pipeline_integration_misc( + self, mock_resolve_dependencies + ): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") selected_groups = ["DynamicResolver"] @@ -1284,13 +1287,14 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): ) pipeline = run.make_pipeline_instance() + mock_resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) self.assertEqual(1, project1.discoveredpackages.count()) @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") def test_scanpipe_resolve_dependencies_pipeline_pypi_integration( - self, resolve_dependencies + self, mock_resolve_dependencies ): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") @@ -1302,7 +1306,7 @@ def test_scanpipe_resolve_dependencies_pipeline_pypi_integration( pipeline = run.make_pipeline_instance() project1.move_input_from(tempfile.mkstemp(suffix="requirements.txt")[1]) - resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) + mock_resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out)