npm: detect UPM packages (Unity)

Unity uses npm packages to distribute various types of features and assets.
See: https://docs.unity3d.com/Manual/upm-manifestPkg.html

The manifest files are using the same name and syntax, but the content, naming convention and of course the upstream repository are not the same.

This is quite annoying because a Unity `package.json` file is gonna be detected as a NPM package from the npmjs registry, but it's actually coming from Unity's repo, which makes a big difference.

Indeed, since both Unity and NodeJS are using NPM and the same package.json files, malicious packages have been pushed to the npmjs registry with the name of Unity packages.

For example the package: `com.unity.scriptablebuildpipeline` is perfectly fine in Unity (https://docs.unity3d.com/Packages/[email protected]/manual/index.html), but is a malware in npmjs (https://www.npmjs.com/package/com.unity.scriptablebuildpipeline).

This means that if ScanCode is being used in a Unity codebase and malicious packages checks are being executed, then we end up with a LOT of scary false positives.
See this advisory: https://github.com/ossf/malicious-packages/blob/cca311974602d1940fab6a98adba611b505cf27d/malicious/npm/com.unity.scriptablebuildpipeline/MAL-2022-2102.json#L13

When matching an inventory against a list of malicious packages, one must take into account the repository from where the packages were fetched, we can't just rely on a canonical purl.

This PR changes the NPM code that parses `package.json` to detect when a file is from UPM and change the urls accordingly so that scanners can take better decisions.

This is probably out of scope for this PR, but maybe we should consider creating a separate purl type for UPM since they don't really share anything with NPM, except the package manager (different languages, different packages, different repositories, different tools etc).

Signed-off-by: Adrien Schildknecht <[email protected]>

src/packagedcode/npm.py

@@ -168,7 +168,12 @@ def walk_npm(cls, resource, codebase, depth=0):
                     yield subchild
 
 
-def get_urls(namespace, name, version, **kwargs):
+def get_urls(namespace, name, version, is_upm=False, **kwargs):
+    if is_upm:
+        return dict(
+            repository_homepage_url='https://docs.unity3d.com/Manual/Packages.html',
+            repository_download_url='https://packages.unity.com',
+        )
     return dict(
         repository_homepage_url=npm_homepage_url(namespace, name, registry='https://www.npmjs.com/package'),
         repository_download_url=npm_download_url(namespace, name, version, registry='https://registry.npmjs.org'),
@@ -189,21 +194,26 @@ def _parse(cls, json_data):
         name = json_data.get('name')
         version = json_data.get('version')
         homepage_url = json_data.get('homepage', '')
+        # Detect Unity packages: https://docs.unity3d.com/Manual/upm-manifestPkg.html
+        is_upm = json_data.get('unity') is not None
 
         # a package.json without name and version can be a private package
 
         if homepage_url and isinstance(homepage_url, list):
             # TODO: should we keep other URLs
             homepage_url = homepage_url[0]
         homepage_url = homepage_url.strip() or None
+        if is_upm and homepage_url is None:
+            homepage_url=f"https://docs.unity3d.com/Packages/{name}@{version}/manual/index.html"
 
         namespace, name = split_scoped_package_name(name)
 
-        urls = get_urls(namespace, name, version)
+        urls = get_urls(namespace, name, version, is_upm)
+
         package = models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
-            primary_language=cls.default_primary_language,
+            primary_language=cls.default_primary_language if not is_upm else None,
             namespace=namespace or None,
             name=name,
             version=version or None,
@@ -242,7 +252,12 @@ def _parse(cls, json_data):
 
         if not package.download_url:
             # Only add a synthetic download URL if there is none from the dist mapping.
-            package.download_url = npm_download_url(package.namespace, package.name, package.version)
+            package.download_url = npm_download_url(
+                package.namespace,
+                package.name,
+                package.version,
+                registry='https://registry.npmjs.org' if not is_upm else 'https://download.packages.unity.com'
+            )
 
         # licenses are a tad special with many different data structures
         lic = json_data.get('license')

tests/packagedcode/data/npm/upm/package/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "com.unity.ml-agents",
+  "displayName": "ML Agents",
+  "version": "3.0.0-exp.1",
+  "unity": "2022.3",
+  "description": "Use state-of-the-art machine learning to create intelligent character behaviors in any Unity environment (games, robotics, film, etc.).",
+  "dependencies": {
+    "com.unity.sentis": "1.2.0-exp.2",
+    "com.unity.modules.imageconversion": "1.0.0",
+    "com.unity.modules.jsonserialize": "1.0.0",
+    "com.unity.modules.physics": "1.0.0"
+  }
+}

tests/packagedcode/data/npm/upm/package/package.json.expected

@@ -0,0 +1,85 @@
+[
+  {
+    "type": "npm",
+    "namespace": null,
+    "name": "com.unity.ml-agents",
+    "version": "3.0.0-exp.1",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": null,
+    "description": "Use state-of-the-art machine learning to create intelligent character behaviors in any Unity environment (games, robotics, film, etc.).",
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": "https://docs.unity3d.com/Packages/[email protected]/manual/index.html",
+    "download_url": "https://download.packages.unity.com/com.unity.ml-agents/-/com.unity.ml-agents-3.0.0-exp.1.tgz",
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "extra_data": {},
+    "dependencies": [
+      {
+        "purl": "pkg:npm/com.unity.sentis",
+        "extracted_requirement": "1.2.0-exp.2",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:npm/com.unity.modules.imageconversion",
+        "extracted_requirement": "1.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:npm/com.unity.modules.jsonserialize",
+        "extracted_requirement": "1.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:npm/com.unity.modules.physics",
+        "extracted_requirement": "1.0.0",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": false,
+        "resolved_package": {},
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": "https://docs.unity3d.com/Manual/Packages.html",
+    "repository_download_url": "https://packages.unity.com",
+    "api_data_url": null,
+    "datasource_id": "npm_package_json",
+    "purl": "pkg:npm/[email protected]"
+  }
+]

tests/packagedcode/test_npm.py

@@ -390,6 +390,12 @@ def test_npm_scan_with_private_package_json_and_yarn_lock(self):
             expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
         )
 
+    def test_npm_upm_package_json(self):
+        # See: https://github.com/Unity-Technologies/ml-agents/com.unity.ml-agents/package.json
+        test_file = self.get_test_loc('npm/upm/package/package.json')
+        expected_loc = self.get_test_loc('npm/upm/package/package.json.expected')
+        packages = npm.NpmPackageJsonHandler.parse(test_file)
+        self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
 
 test_data = [
     (['MIT'], 'mit'),