From 8f374f50623c7a39dae384ae77b9b627e3f63b76 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 19 Jul 2024 13:09:28 -0700 Subject: [PATCH] Update metadata-related code to accommodate fetchcode updates #365 Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 12 +- .../purlcli/expected_metadata_output.json | 378 +++--------------- purldb-toolkit/tests/test_purlcli.py | 76 ---- purldb-toolkit/tests/test_purlcli_live.py | 40 +- 4 files changed, 53 insertions(+), 453 deletions(-) diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 2017f63b..b866d934 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -106,12 +106,12 @@ def collect_metadata(purl): from fetchcode/package.py. """ collected_metadata = [] - for release in list(info(purl)): - if release is None: - continue - release_detail = release.to_dict() - release_detail.move_to_end("purl", last=False) - collected_metadata.append(release_detail) + purl_metadata = info(purl) + if purl_metadata: + for release in list(purl_metadata): + release_detail = release.to_dict() + release_detail.move_to_end("purl", last=False) + collected_metadata.append(release_detail) return collected_metadata diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 794022cb..f4300124 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -12,7 +12,7 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:rubygems/rails" + "pkg:rubygems/pronto-goodcheck" ], "--file": null, "--output": "" @@ -22,42 +22,6 @@ } ], "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, { "purl": "pkg:pypi/fetchcode@0.1.0", "type": "pypi", @@ -238,78 +202,6 @@ "repository_download_url": null, "api_data_url": null }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, { "purl": "pkg:pypi/fetchcode@0.3.0", "type": "pypi", @@ -324,80 +216,6 @@ "parties": [], "keywords": [], "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", "download_url": null, "api_url": "https://pypi.org/pypi/fetchcode/json", "size": null, @@ -421,10 +239,10 @@ "api_data_url": null }, { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", + "purl": "pkg:cargo/banquo@0.1.0", + "type": "cargo", "namespace": null, - "name": "fetchcode", + "name": "banquo", "version": "0.1.0", "qualifiers": {}, "subpath": null, @@ -433,20 +251,20 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": null, + "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", + "api_url": "https://crates.io/api/v1/crates/banquo", "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": "BSD-3-Clause", "notice_text": null, "root_path": null, "dependencies": [], @@ -457,57 +275,24 @@ "api_data_url": null }, { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", + "purl": "pkg:rubygems/pronto-goodcheck@0.2.0", + "type": "rubygems", "namespace": null, - "name": "fetchcode", + "name": "pronto-goodcheck", "version": "0.2.0", "qualifiers": {}, "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.2.0.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.2.0.json", "size": null, "sha1": null, "md5": null, @@ -518,32 +303,34 @@ "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": [ + "MIT" + ], "notice_text": null, "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] }, { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", + "purl": "pkg:rubygems/pronto-goodcheck@0.1.2", + "type": "rubygems", "namespace": null, - "name": "fetchcode", - "version": "0.4.0", + "name": "pronto-goodcheck", + "version": "0.1.2", "qualifiers": {}, "subpath": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", + "repository_download_url": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.2.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.2.json", "size": null, "sha1": null, "md5": null, @@ -554,111 +341,41 @@ "vcs_url": null, "copyright": null, "license_expression": null, - "declared_license": "Apache-2.0", + "declared_license": [ + "MIT" + ], "notice_text": null, "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] }, { - "purl": "pkg:cargo/banquo", - "type": "cargo", + "purl": "pkg:rubygems/pronto-goodcheck@0.1.1", + "type": "rubygems", "namespace": null, - "name": "banquo", - "version": null, + "name": "pronto-goodcheck", + "version": "0.1.1", "qualifiers": {}, "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, + "api_data_url": null, "primary_language": null, "description": null, "release_date": null, "parties": [], "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.1.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.1.json", "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -669,10 +386,7 @@ "root_path": null, "dependencies": [], "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null + "source_packages": [] } ] } diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 3e8650fb..5b898dba 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -58,44 +58,6 @@ def test_metadata_cli_no_input_sources(self): @mock.patch("purldb_toolkit.purlcli.collect_metadata") def test_metadata_details(self, mock_collect_metadata): mock_collect_metadata.return_value = [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"), @@ -237,44 +199,6 @@ def test_metadata_details(self, mock_collect_metadata): } ], "packages": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"), diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index c1be0410..ca14228d 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -50,7 +50,7 @@ def test_metadata_cli(self): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:rubygems/rails", + "pkg:rubygems/pronto-goodcheck", "--output", actual_result_file, ] @@ -152,44 +152,6 @@ def test_metadata_details(self): } ], "packages": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ] - ), OrderedDict( [ ("purl", "pkg:pypi/fetchcode@0.1.0"),