Refactor the group decorator for pipeline steps as optional_step … (

#1442)

* Refactor the `group` decorator for pipeline steps as `optional_step` #1426

Signed-off-by: tdruez <[email protected]>

* Add optional groups explicitly in pipeline tests

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

* Update documentation references of group to option

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

---------

Signed-off-by: tdruez <[email protected]>
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>
Co-authored-by: Ayan Sinha Mahapatra <[email protected]>

CHANGELOG.rst

@@ -18,6 +18,10 @@ v34.9.0 (2024-11-14)
   "policies.yml" files, or global app settings.
   https://github.com/aboutcode-org/scancode.io/issues/386
 
+- Refactor the ``group`` decorator for pipeline steps as ``optional_step``.
+  The steps decorated as optional are not included by default anymore.
+  https://github.com/aboutcode-org/scancode.io/issues/386
+
 - Add a new ``PublishToFederatedCode`` pipeline (addon) to push scan result
   to FederatedCode.
   https://github.com/nexB/scancode.io/pull/1400

aboutcode/pipeline/README.md

@@ -24,11 +24,12 @@ class PrintMessages(BasePipeline):
 PrintMessages().execute()
 ```
 
-### Groups and steps selection
+### Options and steps selection
 
 ```python
 from aboutcode.pipeline import BasePipeline
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
+
 
 class PrintMessages(BasePipeline):
     @classmethod
@@ -38,7 +39,7 @@ class PrintMessages(BasePipeline):
     def step1(self):
         print("Message from step1")
 
-    @group("foo")
+    @optional_step("foo")
     def step2(self):
         print("Message from step2")
 

aboutcode/pipeline/__init__.py

@@ -22,6 +22,7 @@
 
 import logging
 import traceback
+import warnings
 from datetime import datetime
 from datetime import timezone
 from pydoc import getdoc
@@ -53,25 +54,24 @@ def get_steps(cls, groups=None):
         """
         Return the list of steps defined in the ``steps`` class method.
 
-        If the optional ``groups`` parameter is provided, only include steps labeled
-        with groups that intersect with the provided list. If a step has no groups or
-        if ``groups`` is not specified, include the step in the result.
+        By default, all steps decorated with ``optional_step`` are not included.
+        A list of optional steps can be included using the ``groups`` parameter.
         """
         if not callable(cls.steps):
             raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
 
         steps = cls.steps()
+        groups = groups or []
 
         if initial_steps := cls.get_initial_steps():
             steps = (*initial_steps, *steps)
 
-        if groups is not None:
-            steps = tuple(
-                step
-                for step in steps
-                if not getattr(step, "groups", [])
-                or set(getattr(step, "groups")).intersection(groups)
-            )
+        steps = tuple(
+            step
+            for step in steps
+            if not getattr(step, "groups", [])
+            or set(getattr(step, "groups")).intersection(groups)
+        )
 
         return steps
 
@@ -123,7 +123,7 @@ def get_available_groups(cls):
         return sorted(
             set(
                 group_name
-                for step in cls.get_steps()
+                for step in cls.steps()
                 for group_name in getattr(step, "groups", [])
             )
         )
@@ -219,8 +219,8 @@ class BasePipeline(PipelineDefinition, PipelineRun):
     """
 
 
-def group(*groups):
-    """Mark a function as part of a particular group."""
+def optional_step(*groups):
+    """Mark a step function as optional and part of a group."""
 
     def decorator(obj):
         if hasattr(obj, "groups"):
@@ -232,6 +232,17 @@ def decorator(obj):
     return decorator
 
 
+def group(*groups):
+    """Backward compatibility."""
+    warnings.warn(
+        "The `group` decorator is deprecated and will be "
+        "removed in a future release. Use `optional_step` instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return optional_step(*groups)
+
+
 def humanize_time(seconds):
     """Convert the provided ``seconds`` number into human-readable time."""
     message = f"{seconds:.0f} seconds"

docs/built-in-pipelines.rst

@@ -6,6 +6,10 @@ Built-in Pipelines
 Pipelines in ScanCode.io are Python scripts that facilitate code analysis by
 executing a sequence of steps. The platform provides the following built-in pipelines:
 
+.. note::
+    Some pipelines have optional steps which are enabled only when they are
+    selected explicitly.
+
 .. tip::
     If you are unsure which pipeline suits your requirements best, check out the
     :ref:`faq_which_pipeline` section for guidance.

docs/command-line-interface.rst

@@ -103,7 +103,7 @@ Optional arguments:
 - ``--pipeline PIPELINES`` Pipelines names to add on the project.
 
 .. tip::
-    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+    Use the "pipeline_name:option1,option2" syntax to select optional steps:
 
     ``--pipeline map_deploy_to_develop:Java,JavaScript``
 
@@ -230,7 +230,7 @@ add the docker pipeline to your project::
     $ scanpipe add-pipeline --project foo analyze_docker_image
 
 .. tip::
-    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+    Use the "pipeline_name:option1,option2" syntax to select optional steps:
 
     ``--pipeline map_deploy_to_develop:Java,JavaScript``
 
@@ -417,7 +417,7 @@ For example, running the ``inspect_packages`` pipeline on a manifest file:
 
     $ run inspect_packages path/to/package.json > results.json
 
-.. tip:: Use the "pipeline_name:group1,group2" syntax to select steps groups::
+.. tip:: Use the "pipeline_name:option1,option2" syntax to select optional steps::
 
     $ run inspect_packages:StaticResolver package.json > results.json
 

docs/rest-api.rst

@@ -136,7 +136,7 @@ Using cURL:
 
 .. tip::
 
-    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+    Use the "pipeline_name:option1,option2" syntax to select optional steps:
 
     ``"pipeline": "map_deploy_to_develop:Java,JavaScript"``
 
@@ -293,7 +293,7 @@ Data:
     - ``execute_now``: ``true`` or ``false``
 
 .. tip::
-    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+    Use the "pipeline_name:option1,option2" syntax to select optional steps:
 
     ``"pipeline": "map_deploy_to_develop:Java,JavaScript"``
 

scanpipe/management/commands/__init__.py

@@ -229,7 +229,7 @@ def validate_copy_from(copy_from):
 
 def extract_group_from_pipelines(pipelines):
     """
-    Add support for the ":group1,group2" suffix in pipeline data.
+    Add support for the ":option1,option2" suffix in pipeline data.
 
     For example: "map_deploy_to_develop:Java,JavaScript"
     """

scanpipe/management/commands/create-project.py

@@ -42,7 +42,8 @@ def add_arguments(self, parser):
             help=(
                 "Pipelines names to add to the project. "
                 "The pipelines are added and executed based on their given order. "
-                'Groups can be provided using the "pipeline_name:group1,group2" syntax.'
+                'Groups can be provided using the "pipeline_name:option1,option2" '
+                "syntax."
             ),
         )
         parser.add_argument(

scanpipe/management/commands/run.py

@@ -42,7 +42,8 @@ def add_arguments(self, parser):
             help=(
                 "One or more pipeline to run. "
                 "The pipelines executed based on their given order. "
-                'Groups can be provided using the "pipeline_name:group1,group2" syntax.'
+                'Groups can be provided using the "pipeline_name:option1,option2"'
+                " syntax."
             ),
         )
         parser.add_argument(

scanpipe/pipelines/deploy_to_develop.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
@@ -168,35 +168,35 @@ def match_archives_to_purldb(self):
             logger=self.log,
         )
 
-    @group("Java")
+    @optional_step("Java")
     def find_java_packages(self):
         """Find the java package of the .java source files."""
         d2d.find_java_packages(self.project, logger=self.log)
 
-    @group("Java")
+    @optional_step("Java")
     def map_java_to_class(self):
         """Map a .class compiled file to its .java source."""
         d2d.map_java_to_class(project=self.project, logger=self.log)
 
-    @group("Java")
+    @optional_step("Java")
     def map_jar_to_source(self):
         """Map .jar files to their related source directory."""
         d2d.map_jar_to_source(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript(self):
         """
         Map a packed or minified JavaScript, TypeScript, CSS and SCSS
         to its source.
         """
         d2d.map_javascript(project=self.project, logger=self.log)
 
-    @group("Elf")
+    @optional_step("Elf")
     def map_elf(self):
         """Map ELF binaries to their sources."""
         d2d.map_elfs(project=self.project, logger=self.log)
 
-    @group("Go")
+    @optional_step("Go")
     def map_go(self):
         """Map Go binaries to their sources."""
         d2d.map_go_paths(project=self.project, logger=self.log)
@@ -225,22 +225,22 @@ def match_resources_to_purldb(self):
             logger=self.log,
         )
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_post_purldb_match(self):
         """Map minified javascript file based on existing PurlDB match."""
         d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_path(self):
         """Map javascript file based on path."""
         d2d.map_javascript_path(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_colocation(self):
         """Map JavaScript files based on neighborhood file mapping."""
         d2d.map_javascript_colocation(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_thirdparty_npm_packages(self):
         """Map thirdparty package using package.json metadata."""
         d2d.map_thirdparty_npm_packages(project=self.project, logger=self.log)

scanpipe/pipelines/inspect_packages.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import scancode
 
@@ -65,7 +65,7 @@ def scan_for_application_packages(self):
             progress_logger=self.log,
         )
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def resolve_dependencies(self):
         """
         Create packages and dependency relationships from

scanpipe/pipelines/resolve_dependencies.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
@@ -57,7 +57,7 @@ def get_manifest_inputs(self):
         """Locate package manifest files with a supported package resolver."""
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def scan_for_application_packages(self):
         """
         Scan and assemble application packages from package manifests
@@ -70,15 +70,15 @@ def scan_for_application_packages(self):
             progress_logger=self.log,
         )
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def create_packages_and_dependencies(self):
         """
         Create the statically resolved packages and their dependencies
         in the database.
         """
         scancode.process_package_data(self.project, static_resolve=True)
 
-    @group("DynamicResolver")
+    @optional_step("DynamicResolver")
     def get_packages_from_manifest(self):
         """
         Resolve package data from lockfiles/requirement files with package
@@ -91,7 +91,7 @@ def get_packages_from_manifest(self):
             model="get_packages_from_manifest",
         )
 
-    @group("DynamicResolver")
+    @optional_step("DynamicResolver")
     def create_resolved_packages(self):
         """
         Create the dynamically resolved packages and their dependencies

scanpipe/tests/data/d2d/about_files/expected.json

@@ -21,7 +21,9 @@
         {
           "pipeline_name": "map_deploy_to_develop",
           "status": "not_started",
-          "selected_groups": null,
+          "selected_groups": [
+            "Java"
+          ],
           "selected_steps": null,
           "scancodeio_version": "",
           "task_id": null,

scanpipe/tests/data/d2d/flume-ng-node-d2d.json

@@ -21,7 +21,9 @@
         {
           "pipeline_name": "map_deploy_to_develop",
           "status": "not_started",
-          "selected_groups": null,
+          "selected_groups": [
+            "Java"
+          ],
           "selected_steps": null,
           "scancodeio_version": "",
           "task_id": null,

scanpipe/tests/pipelines/with_groups.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines import Pipeline
 
 
@@ -38,17 +38,17 @@ def steps(cls):
             cls.no_groups,
         )
 
-    @group("foo", "bar")
+    @optional_step("foo", "bar")
     def grouped_with_foo_and_bar(self):
         """Step1 doc."""
         pass
 
-    @group("bar")
+    @optional_step("bar")
     def grouped_with_bar(self):
         """Step2 doc."""
         pass
 
-    @group("excluded")
+    @optional_step("excluded")
     def grouped_with_excluded(self):
         """Step2 doc."""
         pass