From a534fcf6b908c81b51fd2f08285fc18069114dba Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 14 Jan 2025 03:26:12 +0530 Subject: [PATCH 1/4] Update package assembly in conda installations Parse conda metadata JSON manifests and use the package data and files information present to improve conda package assembly. Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4083 Signed-off-by: Ayan Sinha Mahapatra --- src/packagedcode/__init__.py | 1 + src/packagedcode/conda.py | 269 ++++++- .../data/conda/assembly-conda-scan.json | 677 ++++++++++++++++++ .../requests-2.32.3-py312h06a4308_1.json | 45 ++ .../requests-2.32.3.dist-info/LICENSE | 1 + .../requests-2.32.3.dist-info/METADATA | 18 + .../site-packages/requests/__init__.py | 1 + .../info/recipe/meta.yaml | 57 ++ .../site-packages/requests/__init__.py | 1 + .../conda-meta/tzdata-2024b-h04d1e81_0.json | 36 + .../conda/conda-meta/tzdata-expected.json | 79 ++ .../meta-yaml/pippy/meta.yaml-scancode.json | 4 +- tests/packagedcode/test_conda.py | 51 ++ 13 files changed, 1237 insertions(+), 3 deletions(-) create mode 100644 tests/packagedcode/data/conda/assembly-conda-scan.json create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests/__init__.py create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml create mode 100644 tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests/__init__.py create mode 100644 tests/packagedcode/data/conda/conda-meta/tzdata-2024b-h04d1e81_0.json create mode 100644 tests/packagedcode/data/conda/conda-meta/tzdata-expected.json diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 778d0434922..9cc46d0e09b 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -73,6 +73,7 @@ cocoapods.PodfileLockHandler, cocoapods.PodfileHandler, + conda.CondaMetaJsonHandler, conda.CondaMetaYamlHandler, conda.CondaYamlHandler, diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index 80b71cbb12b..d2155530688 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -8,6 +8,7 @@ # import io +import json import saneyaml from packageurl import PackageURL @@ -23,7 +24,267 @@ See https://repo.continuum.io/pkgs/free for examples. """ -# TODO: there are likely other package data files for Conda + +class CondaBaseHandler(models.DatafileHandler): + """ + Assemble package data and files present in conda manifests present in the + usual structure of a conda installation. Here the manifests which are + assembled together are: + - Conda metadata JSON (CondaMetaJsonHandler) + - Conda meta.yaml recipe (CondaMetaYamlHandler) + + Example paths for these manifests: + /opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json + /opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml + """ + + @classmethod + def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_package): + + if codebase.has_single_resource: + yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder) + return + + # We do not have any package data detected here + if not resource.package_data: + return + + # If this is a Conda meta.yaml, try to find the corresponding metadata JSON + # and if present, run assembly on the metadata resource + if CondaMetaYamlHandler.is_datafile(resource.location): + conda_meta_json = cls.find_conda_meta_json_resource(resource, codebase) + if conda_meta_json: + package_data_meta_json, = conda_meta_json.package_data + yield from cls.assemble( + package_data=package_data_meta_json, + resource=conda_meta_json, + codebase=codebase, + package_adder=package_adder, + ) + + # corresponding metadata JSON does not exist, so handle this meta.yaml + elif package_data.purl: + package = models.Package.from_package_data( + package_data=package_data, + datafile_path=resource.path, + ) + package.populate_license_fields() + yield package + + CondaMetaYamlHandler.assign_package_to_resources( + package=package, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + yield resource + + return + + # For a conda metadata JSON, try to find the corresponding meta.yaml and + # assemble a single package out of these if it exists + conda_meta_yaml = cls.find_conda_meta_yaml_resource(resource, codebase) + if not package_data.purl: + yield resource + return + + package = models.Package.from_package_data( + package_data=package_data, + datafile_path=resource.path, + ) + if conda_meta_yaml: + conda_meta_yaml_package_data, = conda_meta_yaml.package_data + package.update( + package_data=conda_meta_yaml_package_data, + datafile_path=conda_meta_yaml.path, + ) + cls.assign_package_to_resources( + package=package, + resource=conda_meta_yaml, + codebase=codebase, + package_adder=package_adder, + ) + yield conda_meta_yaml + + package.populate_license_fields() + yield package + + cls.assign_package_to_resources( + package=package, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + + # we yield this as we do not want this further processed + yield resource + + # Get the file paths present in the metadata JSON and assign them to + # the package created from it + extracted_package_dir = package_data.extra_data.get('extracted_package_dir') + files = package_data.extra_data.get('files') + + if not extracted_package_dir or not files: + return + + conda_metadata_dir = resource.parent(codebase) + if not conda_metadata_dir: + return + + conda_root_dir = conda_metadata_dir.parent(codebase) + if not conda_root_dir: + return + + root_path_segment, _, package_dir = extracted_package_dir.rpartition("/pkgs/") + if not conda_root_dir.path.endswith(root_path_segment): + return + + package_dir_path = f"{conda_root_dir.path}/pkgs/{package_dir}" + package_dir_resource = codebase.get_resource(path=package_dir_path) + if package_dir_resource: + cls.assign_package_to_resources( + package=package, + resource=package_dir_resource, + codebase=codebase, + package_adder=package_adder, + ) + + conda_package_path = f"{conda_root_dir.path}/pkgs/{package_dir}.conda" + conda_package_resource = codebase.get_resource(path=conda_package_path) + if conda_package_resource: + cls.assign_package_to_resources( + package=package, + resource=conda_package_resource, + codebase=codebase, + package_adder=package_adder, + ) + + for file_path in files: + full_file_path = f"{conda_root_dir.path}/{file_path}" + file_resource = codebase.get_resource(path=full_file_path) + if file_resource: + cls.assign_package_to_resources( + package=package, + resource=file_resource, + codebase=codebase, + package_adder=package_adder, + ) + + @classmethod + def check_valid_packages_dir_name(cls, package_dir_resource, resource, codebase): + """ + Return the name of the `package_dir_resource`, if it is valid, i.e. + the package (name, version) data present in `resource` matches the + directory name, and the package directory is present in it's usual + location in a conda installation. + """ + package_dir_parent = package_dir_resource.parent(codebase) + + meta_yaml_package_data, = resource.package_data + name = meta_yaml_package_data.get("name") + version = meta_yaml_package_data.get("version") + if f"{name}-{version}" in package_dir_resource.name and ( + package_dir_parent and "pkgs" in package_dir_parent.name + ): + return package_dir_resource.name + + @classmethod + def find_conda_meta_json_resource(cls, resource, codebase): + """ + Given a resource for a conda meta.yaml resource, find if it has any + corresponding metadata JSON located inside the conda-meta/ directory, + and return the resource if they exist, else return None. + """ + package_dir_resource = CondaMetaYamlHandler.get_conda_root(resource, codebase) + if not package_dir_resource or not resource.package_data: + return + + package_dir_name = cls.check_valid_packages_dir_name( + package_dir_resource=package_dir_resource, + resource=resource, + codebase=codebase, + ) + if not package_dir_name: + return + + root_resource = package_dir_resource.parent(codebase).parent(codebase) + if not root_resource: + return + + root_resource_path = root_resource.path + conda_meta_path = f"{root_resource_path}/conda-meta/{package_dir_name}.json" + conda_meta_resource = codebase.get_resource(path=conda_meta_path) + + if conda_meta_resource and conda_meta_resource.package_data: + return conda_meta_resource + + @classmethod + def find_conda_meta_yaml_resource(cls, resource, codebase): + """ + Given a resource for a metadata JSON located inside the conda-meta/ + directory, find if it has any corresponding conda meta.yaml, and return + the resource if they exist, else return None. + """ + package_dir_name, _json, _ = resource.name.rpartition(".json") + parent_resource = resource.parent(codebase) + if not parent_resource and not parent_resource.name == "conda-meta": + return + + root_resource = parent_resource.parent(codebase) + if not root_resource: + return + + root_resource_path = root_resource.path + package_dir_path = f"{root_resource_path}/pkgs/{package_dir_name}/" + package_dir_resource = codebase.get_resource(path=package_dir_path) + if not package_dir_resource: + return + + meta_yaml_path = f"{package_dir_path}info/recipe/meta.yaml" + meta_yaml_resource = codebase.get_resource(path=meta_yaml_path) + if meta_yaml_resource and meta_yaml_resource.package_data: + return meta_yaml_resource + + +class CondaMetaJsonHandler(CondaBaseHandler): + datasource_id = 'conda_meta_json' + path_patterns = ('*conda-meta/*.json',) + default_package_type = 'conda' + default_primary_language = 'Python' + description = 'Conda metadata JSON in rootfs' + documentation_url = 'https://docs.conda.io/' + + @classmethod + def parse(cls, location, package_only=False): + with io.open(location, encoding='utf-8') as loc: + conda_metadata = json.load(loc) + + name = conda_metadata.get('name') + version = conda_metadata.get('version') + extracted_license_statement = conda_metadata.get('license') + download_url = conda_metadata.get('url') + + extra_data_fields = ['requested_spec', 'channel'] + package_file_fields = ['extracted_package_dir', 'files', 'package_tarball_full_path'] + other_package_fields = ['size', 'md5', 'sha256'] + + extra_data = {} + for metadata_field in extra_data_fields + package_file_fields: + extra_data[metadata_field] = conda_metadata.get(metadata_field) + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + version=version, + extracted_license_statement=extracted_license_statement, + download_url=download_url, + extra_data=extra_data, + ) + for package_field in other_package_fields: + package_data[package_field] = conda_metadata.get(package_field) + yield models.PackageData.from_data(package_data, package_only) + class CondaYamlHandler(BaseDependencyFileHandler): datasource_id = 'conda_yaml' @@ -55,7 +316,7 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) -class CondaMetaYamlHandler(models.DatafileHandler): +class CondaMetaYamlHandler(CondaBaseHandler): datasource_id = 'conda_meta_yaml' default_package_type = 'conda' path_patterns = ('*/meta.yaml',) @@ -67,6 +328,9 @@ def get_conda_root(cls, resource, codebase): """ Return a root Resource given a meta.yaml ``resource``. """ + if not resource: + return + # the root is either the parent or further up for yaml stored under # an "info" dir. We support extractcode extraction. # in a source repo it would be in /conda.recipe/meta.yaml @@ -74,6 +338,7 @@ def get_conda_root(cls, resource, codebase): 'info/recipe.tar-extract/recipe/meta.yaml', 'info/recipe/recipe/meta.yaml', 'conda.recipe/meta.yaml', + 'info/recipe/meta.yaml', ) res = resource for pth in paths: diff --git a/tests/packagedcode/data/conda/assembly-conda-scan.json b/tests/packagedcode/data/conda/assembly-conda-scan.json new file mode 100644 index 00000000000..cff1b74e492 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly-conda-scan.json @@ -0,0 +1,677 @@ +{ + "packages": [ + { + "type": "conda", + "namespace": null, + "name": "requests", + "version": "2.32.3", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": "Requests is an elegant and simple HTTP library for Python, built with \u2665.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://requests.readthedocs.io/", + "download_url": "https://repo.anaconda.com/pkgs/main/linux-64/requests-2.32.3-py312h06a4308_1.conda", + "size": 125693, + "sha1": null, + "md5": "8cc2fc3e2198c2efe6cd890a7684a16a", + "sha256": "940b9ae3f0b64e7ee51dfbdcfcffc674a447e5592f8a66d8064cd505fc122b78", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/psf/requests", + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + }, + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "Apache-2.0", + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "requested_spec": "defaults/linux-64::requests==2.32.3=py312h06a4308_1[md5=8cc2fc3e2198c2efe6cd890a7684a16a]", + "channel": "https://repo.anaconda.com/pkgs/main/linux-64", + "extracted_package_dir": "/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1", + "files": [ + "lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE", + "lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "lib/python3.12/site-packages/requests/__init__.py" + ], + "package_tarball_full_path": "opt/conda/pkgs/requests-2.32.3-py312h06a4308_1.conda" + }, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json", + "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml" + ], + "datasource_ids": [ + "conda_meta_json", + "conda_meta_yaml" + ], + "purl": "pkg:conda/requests@2.32.3" + }, + { + "type": "pypi", + "namespace": null, + "name": "requests", + "version": "2.32.3", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Python HTTP for Humans.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Kenneth Reitz", + "email": "me@kennethreitz.org", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3" + ], + "homepage_url": "https://requests.readthedocs.io", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/psf/requests", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: Apache-2.0\n", + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "Documentation": "https://requests.readthedocs.io" + }, + "repository_homepage_url": "https://pypi.org/project/requests", + "repository_download_url": "https://pypi.org/packages/source/r/requests/requests-2.32.3.tar.gz", + "api_data_url": "https://pypi.org/pypi/requests/2.32.3/json", + "package_uid": "pkg:pypi/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA" + ], + "datasource_ids": [ + "pypi_wheel_metadata" + ], + "purl": "pkg:pypi/requests@2.32.3" + } + ], + "dependencies": [ + { + "purl": "pkg:pypi/charset-normalizer", + "extracted_requirement": "<4,>=2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:pypi/charset-normalizer?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "datasource_id": "pypi_wheel_metadata" + } + ], + "files": [ + { + "path": "assembly", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/conda-meta", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json", + "type": "file", + "package_data": [ + { + "type": "conda", + "namespace": null, + "name": "requests", + "version": "2.32.3", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://repo.anaconda.com/pkgs/main/linux-64/requests-2.32.3-py312h06a4308_1.conda", + "size": 125693, + "sha1": null, + "md5": "8cc2fc3e2198c2efe6cd890a7684a16a", + "sha256": "940b9ae3f0b64e7ee51dfbdcfcffc674a447e5592f8a66d8064cd505fc122b78", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "Apache-2.0", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "requested_spec": "defaults/linux-64::requests==2.32.3=py312h06a4308_1[md5=8cc2fc3e2198c2efe6cd890a7684a16a]", + "channel": "https://repo.anaconda.com/pkgs/main/linux-64", + "extracted_package_dir": "/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1", + "files": [ + "lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE", + "lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "lib/python3.12/site-packages/requests/__init__.py" + ], + "package_tarball_full_path": "opt/conda/pkgs/requests-2.32.3-py312h06a4308_1.conda" + }, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_json", + "purl": "pkg:conda/requests@2.32.3" + } + ], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages/requests", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": "requests", + "version": "2.32.3", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Python HTTP for Humans.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Kenneth Reitz", + "email": "me@kennethreitz.org", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3" + ], + "homepage_url": "https://requests.readthedocs.io", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/psf/requests", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: Apache-2.0\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "Documentation": "https://requests.readthedocs.io" + }, + "dependencies": [ + { + "purl": "pkg:pypi/charset-normalizer", + "extracted_requirement": "<4,>=2", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pypi.org/project/requests", + "repository_download_url": "https://pypi.org/packages/source/r/requests/requests-2.32.3.tar.gz", + "api_data_url": "https://pypi.org/pypi/requests/2.32.3/json", + "datasource_id": "pypi_wheel_metadata", + "purl": "pkg:pypi/requests@2.32.3" + } + ], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:pypi/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/lib/python3.12/site-packages/requests/__init__.py", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "type": "file", + "package_data": [ + { + "type": "conda", + "namespace": null, + "name": "requests", + "version": "2.32.3", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": "Requests is an elegant and simple HTTP library for Python, built with \u2665.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://requests.readthedocs.io/", + "download_url": "https://pypi.io/packages/source/r/requests/requests-2.32.3.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/psf/requests", + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "matched_text": "Apache-2.0" + } + ], + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "Apache-2.0", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:conda/zlib", + "extracted_requirement": "1.2.13 h5eee18b_1", + "scope": "host", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/certifi", + "extracted_requirement": ">=2017.4.17", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/chardet", + "extracted_requirement": ">=3.0.2,<6", + "scope": "run_constrained", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_yaml", + "purl": "pkg:conda/requests@2.32.3" + } + ], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests", + "type": "directory", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests/__init__.py", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json b/tests/packagedcode/data/conda/assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json new file mode 100644 index 00000000000..085fad49dd6 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/conda-meta/requests-2.32.3-py312h06a4308_1.json @@ -0,0 +1,45 @@ +{ + "arch": "x86_64", + "build": "py312h06a4308_1", + "build_number": 1, + "channel": "https://repo.anaconda.com/pkgs/main/linux-64", + "constrains": [ + "pysocks >=1.5.6,!=1.5.7", + "chardet >=3.0.2,<6" + ], + "depends": [ + "certifi >=2017.4.17", + "charset-normalizer >=2,<4", + "idna >=2.5,<4", + "python >=3.12,<3.13.0a0", + "urllib3 >=1.21.1,<3" + ], + "extracted_package_dir": "/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1", + "files": [ + "lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE", + "lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA", + "lib/python3.12/site-packages/requests/__init__.py" + ], + "fn": "requests-2.32.3-py312h06a4308_1.conda", + "license": "Apache-2.0", + "license_family": "Apache", + "link": { + "source": "opt/conda/pkgs/requests-2.32.3-py312h06a4308_1", + "type": 1 + }, + "md5": "8cc2fc3e2198c2efe6cd890a7684a16a", + "name": "requests", + "package_tarball_full_path": "opt/conda/pkgs/requests-2.32.3-py312h06a4308_1.conda", + "paths_data": { + "paths": [], + "paths_version": 1 + }, + "platform": "linux", + "requested_spec": "defaults/linux-64::requests==2.32.3=py312h06a4308_1[md5=8cc2fc3e2198c2efe6cd890a7684a16a]", + "sha256": "940b9ae3f0b64e7ee51dfbdcfcffc674a447e5592f8a66d8064cd505fc122b78", + "size": 125693, + "subdir": "linux-64", + "timestamp": 1730999176000, + "url": "https://repo.anaconda.com/pkgs/main/linux-64/requests-2.32.3-py312h06a4308_1.conda", + "version": "2.32.3" +} \ No newline at end of file diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE new file mode 100644 index 00000000000..2e85972ece4 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/LICENSE @@ -0,0 +1 @@ +Apache License Version 2.0 \ No newline at end of file diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA new file mode 100644 index 00000000000..1166fc320a1 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests-2.32.3.dist-info/METADATA @@ -0,0 +1,18 @@ +Metadata-Version: 2.1 +Name: requests +Version: 2.32.3 +Summary: Python HTTP for Humans. +Home-page: https://requests.readthedocs.io +Author: Kenneth Reitz +Author-email: me@kennethreitz.org +License: Apache-2.0 +Project-URL: Documentation, https://requests.readthedocs.io +Project-URL: Source, https://github.com/psf/requests +Classifier: Development Status :: 5 - Production/Stable +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=3.8 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: charset-normalizer<4,>=2 diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests/__init__.py b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/lib/python3.12/site-packages/requests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml b/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml new file mode 100644 index 00000000000..7615c560abe --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml @@ -0,0 +1,57 @@ +# This file created by conda-build 24.1.2 +# meta.yaml template originally from: +# /feedstock/recipe, last modified Thu Nov 7 17:05:12 2024 +# ------------------------------------------------ + +package: + name: requests + version: 2.32.3 +source: + sha256: 55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 + url: https://pypi.io/packages/source/r/requests/requests-2.32.3.tar.gz +requirements: + host: + - zlib 1.2.13 h5eee18b_1 + run: + - certifi >=2017.4.17 + run_constrained: + - chardet >=3.0.2,<6 +test: + commands: + - pip check + - conda create -v --dry-run -n requests-test numpy + imports: + - requests + requires: + - conda + - pip +about: + description: 'Requests is the only Non-GMO HTTP library for Python, safe for human + + consumption. + + ' + dev_url: https://github.com/psf/requests + doc_url: https://requests.readthedocs.io/ + home: https://requests.readthedocs.io/ + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: "Requests is an elegant and simple HTTP library for Python, built with\ + \ \u2665." +extra: + copy_test_source_files: true + final: true + flow_run_id: a6780417-397a-4b4b-9b26-7ad861f84324 + recipe-maintainers: + - carlodri + - jakirkham + - kalefranz + - mcg1969 + - mingwandroid + - msarahan + - ocefpaf + - pelson + - sigmavirus24 + remote_url: git@github.com:AnacondaRecipes/requests-feedstock.git + sha: 997bb0da1a210a5aeef4073b32f674eaf1f8eaca \ No newline at end of file diff --git a/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests/__init__.py b/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/packagedcode/data/conda/assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/lib/python3.12/site-packages/requests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/packagedcode/data/conda/conda-meta/tzdata-2024b-h04d1e81_0.json b/tests/packagedcode/data/conda/conda-meta/tzdata-2024b-h04d1e81_0.json new file mode 100644 index 00000000000..bae5927378b --- /dev/null +++ b/tests/packagedcode/data/conda/conda-meta/tzdata-2024b-h04d1e81_0.json @@ -0,0 +1,36 @@ +{ + "build": "h04d1e81_0", + "build_number": 0, + "channel": "https://repo.anaconda.com/pkgs/main/noarch", + "constrains": [], + "depends": [], + "extracted_package_dir": "/opt/conda/pkgs/tzdata-2024b-h04d1e81_0", + "files": [ + "share/zoneinfo/Asia/Kolkata", + "share/zoneinfo/tzdata.zi", + "share/zoneinfo/zone.tab" + ], + "fn": "tzdata-2024b-h04d1e81_0.conda", + "license": "CC-PDDC OR BSD-3-Clause", + "license_family": "BSD", + "link": { + "source": "/opt/conda/pkgs/tzdata-2024b-h04d1e81_0", + "type": 1 + }, + "md5": "9be694715c6a65f9631bb1b242125e9d", + "name": "tzdata", + "noarch": "generic", + "package_tarball_full_path": "/opt/conda/pkgs/tzdata-2024b-h04d1e81_0.conda", + "package_type": "noarch_generic", + "paths_data": { + "paths": [], + "paths_version": 1 + }, + "requested_spec": "defaults/noarch::tzdata==2024b=h04d1e81_0[md5=9be694715c6a65f9631bb1b242125e9d]", + "sha256": "9fdd287b55be4c475789a69d3b94cdb73f756583a6d7306da1706e43eee573da", + "size": 117432, + "subdir": "noarch", + "timestamp": 1728062827000, + "url": "https://repo.anaconda.com/pkgs/main/noarch/tzdata-2024b-h04d1e81_0.conda", + "version": "2024b" +} \ No newline at end of file diff --git a/tests/packagedcode/data/conda/conda-meta/tzdata-expected.json b/tests/packagedcode/data/conda/conda-meta/tzdata-expected.json new file mode 100644 index 00000000000..84141ae2dab --- /dev/null +++ b/tests/packagedcode/data/conda/conda-meta/tzdata-expected.json @@ -0,0 +1,79 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "tzdata", + "version": "2024b", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://repo.anaconda.com/pkgs/main/noarch/tzdata-2024b-h04d1e81_0.conda", + "size": 117432, + "sha1": null, + "md5": "9be694715c6a65f9631bb1b242125e9d", + "sha256": "9fdd287b55be4c475789a69d3b94cdb73f756583a6d7306da1706e43eee573da", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "cc-pd OR bsd-new", + "declared_license_expression_spdx": "CC-PDDC OR BSD-3-Clause", + "license_detections": [ + { + "license_expression": "cc-pd OR bsd-new", + "license_expression_spdx": "CC-PDDC OR BSD-3-Clause", + "matches": [ + { + "license_expression": "cc-pd OR bsd-new", + "license_expression_spdx": "CC-PDDC OR BSD-3-Clause", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 6, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-cc_pd_or_bsd_new-7ed06e8faddcea41dbd5ba6b6a1103da7451d4c1", + "rule_url": null, + "matched_text": "CC-PDDC OR BSD-3-Clause" + } + ], + "identifier": "cc_pd_or_bsd_new-59c31305-cd29-a988-bea8-38f97ce1950f" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "CC-PDDC OR BSD-3-Clause", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "requested_spec": "defaults/noarch::tzdata==2024b=h04d1e81_0[md5=9be694715c6a65f9631bb1b242125e9d]", + "channel": "https://repo.anaconda.com/pkgs/main/noarch", + "extracted_package_dir": "/opt/conda/pkgs/tzdata-2024b-h04d1e81_0", + "files": [ + "share/zoneinfo/Asia/Kolkata", + "share/zoneinfo/tzdata.zi", + "share/zoneinfo/zone.tab" + ], + "package_tarball_full_path": "/opt/conda/pkgs/tzdata-2024b-h04d1e81_0.conda" + }, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_json", + "purl": "pkg:conda/tzdata@2024b" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json index d41b17c4418..d55dac7f9a7 100644 --- a/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json +++ b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json @@ -376,7 +376,9 @@ "purl": "pkg:conda/pippy@0.1.0" } ], - "for_packages": [], + "for_packages": [ + "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], "scan_errors": [] } ] diff --git a/tests/packagedcode/test_conda.py b/tests/packagedcode/test_conda.py index 05df09149a6..4753beefef5 100644 --- a/tests/packagedcode/test_conda.py +++ b/tests/packagedcode/test_conda.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import attr import os from commoncode.resource import Codebase @@ -113,3 +114,53 @@ def test_parse_conda_yaml_simple_dependencies(self): package = conda.CondaYamlHandler.parse(test_file) expected_loc = self.get_test_loc('conda/conda-yaml/ringer/environment.yaml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_conda_get_conda_meta_json(self): + meta_yaml_path = 'conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml' + conda_meta_json_path = 'conda/conda-meta/requests-2.32.3-py312h06a4308_1.json' + + test_dir = self.get_test_loc('conda/assembly/opt/conda/') + resource_attributes = dict(package_data=attr.ib(default=attr.Factory(list), repr=False),) + codebase = Codebase(location=test_dir, resource_attributes=resource_attributes) + + package_data = [{'name': 'requests', 'version':'2.32.3'}] + meta_yaml_resource = codebase.get_resource(path=meta_yaml_path) + setattr(meta_yaml_resource, 'package_data', package_data) + codebase.save_resource(meta_yaml_resource) + conda_meta_json_resource = codebase.get_resource(path=conda_meta_json_path) + setattr(conda_meta_json_resource, 'package_data', package_data) + codebase.save_resource(conda_meta_json_resource) + + meta_json = conda.CondaBaseHandler.find_conda_meta_json_resource(meta_yaml_resource, codebase) + assert meta_json.path == conda_meta_json_path + + meta_yaml = conda.CondaBaseHandler.find_conda_meta_yaml_resource(conda_meta_json_resource, codebase) + assert meta_yaml.path == meta_yaml_path + + def test_conda_pkgs_meta_yaml_root_dir(self): + meta_yaml_path = 'conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml' + root_path = 'conda/pkgs/requests-2.32.3-py312h06a4308_1' + test_dir = self.get_test_loc('conda/assembly/opt/conda/') + codebase = Codebase(test_dir) + resource = codebase.get_resource(path=meta_yaml_path) + proot = conda.CondaMetaYamlHandler.get_conda_root(resource, codebase) + assert proot.path == root_path + + def test_parse_is_datafile_conda_meta_package_with_files(self): + test_file = self.get_test_loc('conda/conda-meta/tzdata-2024b-h04d1e81_0.json') + assert conda.CondaMetaJsonHandler.is_datafile(test_file) + + def test_parse_conda_meta_package_with_files(self): + test_file = self.get_test_loc('conda/conda-meta/tzdata-2024b-h04d1e81_0.json') + package = conda.CondaMetaJsonHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/conda-meta/tzdata-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_meta_yaml_conda_meta_assemble_from_rootfs(self): + test_location = self.get_test_loc('conda/assembly/') + result_file = self.get_temp_file('results.json') + run_scan_click(['--package', test_location, '--json', result_file]) + expected_file = self.get_test_loc('conda/assembly-conda-scan.json') + check_json_scan( + expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES + ) From dce5b2a455eaa55bb6e493b48647403bf49f9d43 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 14 Jan 2025 03:39:47 +0530 Subject: [PATCH 2/4] Replace deprecated macos-12 runner with macos-14 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 28389ccc9e7..7f3f89f6e4f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -233,8 +233,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos12_cpython_latest_from_pip - image_name: macos-12 + job_name: macos14_cpython_latest_from_pip + image_name: macos-14 python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py From 20663f45f0cc0743e179117c28914387598e177b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 14 Jan 2025 15:16:58 +0530 Subject: [PATCH 3/4] Report top level dependencies properly for conda packages Signed-off-by: Ayan Sinha Mahapatra --- src/packagedcode/conda.py | 17 +++++++ .../data/conda/assembly-conda-scan.json | 45 +++++++++++++++++++ .../data/plugin/plugins_list_linux.txt | 7 +++ 3 files changed, 69 insertions(+) diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index d2155530688..855d4a10039 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -71,6 +71,15 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ package.populate_license_fields() yield package + dependent_packages = package_data.dependencies + if dependent_packages: + yield from models.Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=resource.path, + datasource_id=package_data.datasource_id, + package_uid=package.package_uid, + ) + CondaMetaYamlHandler.assign_package_to_resources( package=package, resource=resource, @@ -104,6 +113,14 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ codebase=codebase, package_adder=package_adder, ) + meta_yaml_package_data = models.PackageData.from_dict(conda_meta_yaml_package_data) + if meta_yaml_package_data.dependencies: + yield from models.Dependency.from_dependent_packages( + dependent_packages=meta_yaml_package_data.dependencies, + datafile_path=conda_meta_yaml.path, + datasource_id=meta_yaml_package_data.datasource_id, + package_uid=package.package_uid, + ) yield conda_meta_yaml package.populate_license_fields() diff --git a/tests/packagedcode/data/conda/assembly-conda-scan.json b/tests/packagedcode/data/conda/assembly-conda-scan.json index cff1b74e492..97cecaa7c8f 100644 --- a/tests/packagedcode/data/conda/assembly-conda-scan.json +++ b/tests/packagedcode/data/conda/assembly-conda-scan.json @@ -193,6 +193,51 @@ } ], "dependencies": [ + { + "purl": "pkg:conda/zlib", + "extracted_requirement": "1.2.13 h5eee18b_1", + "scope": "host", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/zlib?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/certifi", + "extracted_requirement": ">=2017.4.17", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/certifi?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/chardet", + "extracted_requirement": ">=3.0.2,<6", + "scope": "run_constrained", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/chardet?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/requests@2.32.3?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "assembly/opt/conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml", + "datasource_id": "conda_meta_yaml" + }, { "purl": "pkg:pypi/charset-normalizer", "extracted_requirement": "<4,>=2", diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index 33886e87222..e24512dfd91 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -195,6 +195,13 @@ Package type: conan description: conan recipe path_patterns: '*/conanfile.py' -------------------------------------------- +Package type: conda + datasource_id: conda_meta_json + documentation URL: https://docs.conda.io/ + primary language: Python + description: Conda metadata JSON in rootfs + path_patterns: '*conda-meta/*.json' +-------------------------------------------- Package type: conda datasource_id: conda_meta_yaml documentation URL: https://docs.conda.io/ From 32f8797f097d96521dbcc801c0e84ecf18f508a1 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 14 Jan 2025 19:20:28 +0530 Subject: [PATCH 4/4] Refactor conda package assembly Signed-off-by: Ayan Sinha Mahapatra --- src/packagedcode/conda.py | 141 +++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 46 deletions(-) diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index 855d4a10039..29bee04eac3 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -63,36 +63,16 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ ) # corresponding metadata JSON does not exist, so handle this meta.yaml - elif package_data.purl: - package = models.Package.from_package_data( + else: + yield from cls.assemble_from_meta_yaml_only( package_data=package_data, - datafile_path=resource.path, - ) - package.populate_license_fields() - yield package - - dependent_packages = package_data.dependencies - if dependent_packages: - yield from models.Dependency.from_dependent_packages( - dependent_packages=dependent_packages, - datafile_path=resource.path, - datasource_id=package_data.datasource_id, - package_uid=package.package_uid, - ) - - CondaMetaYamlHandler.assign_package_to_resources( - package=package, resource=resource, codebase=codebase, package_adder=package_adder, ) - yield resource return - # For a conda metadata JSON, try to find the corresponding meta.yaml and - # assemble a single package out of these if it exists - conda_meta_yaml = cls.find_conda_meta_yaml_resource(resource, codebase) if not package_data.purl: yield resource return @@ -101,27 +81,12 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ package_data=package_data, datafile_path=resource.path, ) - if conda_meta_yaml: - conda_meta_yaml_package_data, = conda_meta_yaml.package_data - package.update( - package_data=conda_meta_yaml_package_data, - datafile_path=conda_meta_yaml.path, - ) - cls.assign_package_to_resources( - package=package, - resource=conda_meta_yaml, - codebase=codebase, - package_adder=package_adder, - ) - meta_yaml_package_data = models.PackageData.from_dict(conda_meta_yaml_package_data) - if meta_yaml_package_data.dependencies: - yield from models.Dependency.from_dependent_packages( - dependent_packages=meta_yaml_package_data.dependencies, - datafile_path=conda_meta_yaml.path, - datasource_id=meta_yaml_package_data.datasource_id, - package_uid=package.package_uid, - ) - yield conda_meta_yaml + yield from cls.get_and_assmeble_from_meta_yaml( + package=package, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) package.populate_license_fields() yield package @@ -136,8 +101,27 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ # we yield this as we do not want this further processed yield resource - # Get the file paths present in the metadata JSON and assign them to - # the package created from it + cls.assign_packages_to_resources_from_metadata_json( + package=package, + package_data=package_data, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + + @classmethod + def assign_packages_to_resources_from_metadata_json( + cls, + package, + package_data, + resource, + codebase, + package_adder=models.add_to_package, + ): + """ + Get the file paths present in the `package_data` of a metadata JSON `resource` + and assign them to the `package` created from the manifest. + """ extracted_package_dir = package_data.extra_data.get('extracted_package_dir') files = package_data.extra_data.get('files') @@ -175,7 +159,7 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ codebase=codebase, package_adder=package_adder, ) - + for file_path in files: full_file_path = f"{conda_root_dir.path}/{file_path}" file_resource = codebase.get_resource(path=full_file_path) @@ -187,6 +171,71 @@ def assemble(cls, package_data, resource, codebase, package_adder=models.add_to_ package_adder=package_adder, ) + @classmethod + def get_and_assmeble_from_meta_yaml(cls, package, resource, codebase, package_adder=models.add_to_package): + """ + For a conda metadata JSON `resource`, try to find the corresponding meta.yaml and + update the `package` from it. Also yield dependencies present in the meta.yaml, + and the `resource` to complete assembling from this manifest. + """ + conda_meta_yaml = cls.find_conda_meta_yaml_resource(resource, codebase) + + if conda_meta_yaml: + conda_meta_yaml_package_data, = conda_meta_yaml.package_data + package.update( + package_data=conda_meta_yaml_package_data, + datafile_path=conda_meta_yaml.path, + ) + cls.assign_package_to_resources( + package=package, + resource=conda_meta_yaml, + codebase=codebase, + package_adder=package_adder, + ) + meta_yaml_package_data = models.PackageData.from_dict(conda_meta_yaml_package_data) + if meta_yaml_package_data.dependencies: + yield from models.Dependency.from_dependent_packages( + dependent_packages=meta_yaml_package_data.dependencies, + datafile_path=conda_meta_yaml.path, + datasource_id=meta_yaml_package_data.datasource_id, + package_uid=package.package_uid, + ) + + yield conda_meta_yaml + + @classmethod + def assemble_from_meta_yaml_only(cls, package_data, resource, codebase, package_adder=models.add_to_package): + """ + Assemble and yeild package, dependencies and the meta YAML `resource` from + it's `package_data`, and also assign resources to the package. + """ + if not package_data.purl: + return + + package = models.Package.from_package_data( + package_data=package_data, + datafile_path=resource.path, + ) + package.populate_license_fields() + yield package + + dependent_packages = package_data.dependencies + if dependent_packages: + yield from models.Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=resource.path, + datasource_id=package_data.datasource_id, + package_uid=package.package_uid, + ) + + CondaMetaYamlHandler.assign_package_to_resources( + package=package, + resource=resource, + codebase=codebase, + package_adder=package_adder, + ) + yield resource + @classmethod def check_valid_packages_dir_name(cls, package_dir_resource, resource, codebase): """