diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index dfc185e2..c08a86de 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -8,8 +8,11 @@ # import json +import logging +import os import re from importlib.metadata import version +from pathlib import Path import click import requests @@ -18,8 +21,6 @@ from packageurl import PackageURL from packageurl.contrib import purl2url -from packagedb.package_managers import VERSION_API_CLASSES_BY_PACKAGE_TYPE - @click.group() def purlcli(): @@ -84,14 +85,9 @@ def get_metadata_details(purls, output, file, unique, command_name): metadata_warnings = {} - input_purls = [] - normalized_purls = [] - if unique: - input_purls, normalized_purls = normalize_purls( - purls, input_purls, normalized_purls - ) - else: - input_purls = purls + input_purls, normalized_purls = normalize_purls(purls, unique) + + clear_log_file() for purl in input_purls: purl = purl.strip() @@ -133,10 +129,10 @@ def check_metadata_purl(purl): `warnings` field of the `header` section of the JSON object returned by the `metadata` command. """ - check_validation = validate_purls([purl]) + check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation[0] + results = check_validation if results["valid"] == False: return "not_valid" @@ -159,16 +155,20 @@ def check_metadata_purl(purl): return "not_in_upstream_repo" -def normalize_purls(purls, input_purls, normalized_purls): - for purl in purls: - input_purl = purl - purl = purl.strip() - purl = re.split("[@,?,#,]+", purl)[0] - normalized_purl = purl - - normalized_purls.append((input_purl, normalized_purl)) - if normalized_purl not in input_purls: - input_purls.append(normalized_purl) +def normalize_purls(purls, unique): + input_purls = [] + normalized_purls = [] + if unique: + for purl in purls: + input_purl = purl + purl = purl.strip() + purl = re.split("[@,?,#,]+", purl)[0] + normalized_purl = purl + normalized_purls.append((input_purl, normalized_purl)) + if normalized_purl not in input_purls: + input_purls.append(normalized_purl) + else: + input_purls = purls return input_purls, normalized_purls @@ -219,7 +219,7 @@ def construct_headers( headers_content["options"] = options headers_content["purls"] = purls - if (command_name in ["metadata", "urls"]) and unique: + if (command_name in ["metadata", "urls", "validate", "versions"]) and unique: for purl in normalized_purls: if purl[0] != purl[1]: warnings.append(f"input PURL: '{purl[0]}' normalized to '{purl[1]}'") @@ -229,6 +229,7 @@ def construct_headers( continue warning_text = { + "error_fetching_purl": f"'error fetching {purl}'", "validation_error": f"'{purl}' encountered a validation error", "not_valid": f"'{purl}' not valid", "valid_but_not_supported": f"'{purl}' not supported with `{command_name}` command", @@ -236,24 +237,18 @@ def construct_headers( "not_in_upstream_repo": f"'{purl}' does not exist in the upstream repo", } - # `metadata` warnings: - if command_name == "metadata": - purl_warning = purl_warnings.get(purl, None) - if purl_warning: - warnings.append(warning_text[purl_warning]) - print(warning_text[purl_warning]) - continue - - # `urls` warnings: - if command_name == "urls": + if command_name in ["metadata", "urls", "validate", "versions"]: purl_warning = purl_warnings.get(purl, None) if purl_warning: warnings.append(warning_text[purl_warning]) print(warning_text[purl_warning]) continue - # add `versions` warnings here - # it's not yet clear whether `validate` will have any similar warnings + log_file = Path("purldb-toolkit/src/purldb_toolkit/app.log") + if log_file.is_file(): + with open(log_file, "r") as f: + for line in f: + errors.append(line) headers_content["errors"] = errors headers_content["warnings"] = warnings @@ -324,14 +319,9 @@ def get_urls_details(purls, output, file, unique, head, command_name): urls_warnings = {} - input_purls = [] - normalized_purls = [] - if unique: - input_purls, normalized_purls = normalize_purls( - purls, input_purls, normalized_purls - ) - else: - input_purls = purls + input_purls, normalized_purls = normalize_purls(purls, unique) + + clear_log_file() for purl in input_purls: url_detail = {} @@ -451,10 +441,10 @@ def check_urls_purl(purl): or its type is not supported (or not fully supported) by `urls`, or it does not exist in the upstream repo. """ - check_validation = validate_purls([purl]) + check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation[0] + results = check_validation if results["valid"] == False: return "not_valid" @@ -512,7 +502,6 @@ def check_urls_purl(purl): return "valid_but_not_fully_supported" -# Not yet converted to a SCTK-like data structure. @purlcli.command(name="validate") @click.option( "--purl", @@ -534,22 +523,100 @@ def check_urls_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -def validate(purls, output, file): +@click.option( + "--unique", + is_flag=True, + required=False, + help="Return data only for unique PURLs.", +) +def validate(purls, output, file, unique): """ - Check the syntax of one or more PURLs. + Check the syntax and upstream repo status of one or more PURLs. """ check_for_duplicate_input_sources(purls, file) if file: purls = file.read().splitlines(False) - validated_purls = validate_purls(purls) + context = click.get_current_context() + command_name = context.command.name + + validated_purls = get_validate_details(purls, output, file, unique, command_name) json.dump(validated_purls, output, indent=4) -def validate_purls(purls): +def get_validate_details(purls, output, file, unique, command_name): + """ + Return a dictionary containing validation data for each PURL in the `purls` + input list. + """ + validate_details = {} + validate_details["headers"] = [] + + validate_warnings = {} + + input_purls, normalized_purls = normalize_purls(purls, unique) + + validate_details["packages"] = [] + + clear_log_file() + + for purl in input_purls: + purl = purl.strip() + if not purl: + continue + + validated_purl = check_validate_purl(purl) + + if command_name == "urls" and validated_purl in [ + "validation_error", + "not_valid", + "valid_but_not_supported", + "not_in_upstream_repo", + ]: + validate_warnings[purl] = validated_purl + continue + + if validated_purl: + validate_details["packages"].append(validate_purl(purl)) + + validate_details["headers"] = construct_headers( + purls=purls, + output=output, + file=file, + command_name=command_name, + normalized_purls=normalized_purls, + unique=unique, + purl_warnings=validate_warnings, + ) + + return validate_details + + +def check_validate_purl(purl): + """ + As applicable, return a variable indicating that the input PURL is + valid/invalid or does not exist in the upstream repo. + """ + check_validation = validate_purl(purl) + if check_validation is None: + return "validation_error" + results = check_validation + + if results["valid"] == False: + return "not_valid" + + if results["exists"] == False: + return "not_in_upstream_repo" + + if results["exists"] == True: + return check_validation + + +def validate_purl(purl): """ - Return a JSON object containing data regarding the validity of the input PURL. + Return a JSON object containing data from the PurlDB `validate` endpoint + regarding the validity of the input PURL. Based on packagedb.package_managers VERSION_API_CLASSES_BY_PACKAGE_TYPE and packagedb/api.py class PurlValidateViewSet(viewsets.ViewSet) @@ -567,31 +634,36 @@ def validate_purls(purls): "nuget", "pypi", """ + logger = logging.getLogger(__name__) + api_query = "https://public.purldb.io/api/validate/" - validated_purls = [] - for purl in purls: - purl = purl.strip() - if not purl: - continue - request_body = {"purl": purl, "check_existence": True} - response = requests.get(api_query, params=request_body) - try: - results = response.json() - # print(f"response - {response}") - # print(f"response.text - {response.text}") - # print(f"response.json() - {response.json()}") - except Exception as e: - print(f"'validate' endpoint error for '{purl}': {e}") - # print(f"response - {response}") - # print(f"response.text - {response.text}") - # print(f"response.json() - {response.json()}") - return - validated_purls.append(results) - - return validated_purls - - -# Not yet converted to a SCTK-like data structure. + request_body = {"purl": purl, "check_existence": True} + + try: + response = requests.get(api_query, params=request_body).json() + + except json.decoder.JSONDecodeError as e: + + print(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") + + logging.basicConfig( + filename="purldb-toolkit/src/purldb_toolkit/app.log", + level=logging.ERROR, + format="%(levelname)s - %(message)s", + filemode="w", + ) + + logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") + + except Exception as e: + print(f"'validate' endpoint error for '{purl}': {e}") + + else: + if response is None: + print(f"'{purl}' -- response.status_code for None = {response.status_code}") + return response + + @purlcli.command(name="versions") @click.option( "--purl", @@ -613,7 +685,13 @@ def validate_purls(purls): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -def get_versions(purls, output, file): +@click.option( + "--unique", + is_flag=True, + required=False, + help="Return data only for unique PURLs.", +) +def get_versions(purls, output, file, unique): """ Given one or more PURLs, return a list of all known versions for each PURL. """ @@ -625,78 +703,86 @@ def get_versions(purls, output, file): context = click.get_current_context() command_name = context.command.name - purl_versions = list_versions(purls, output, file, command_name) + purl_versions = get_versions_details(purls, output, file, unique, command_name) json.dump(purl_versions, output, indent=4) -# construct_headers() has not yet been implemented for this `versions` command -# -- or for the `validate` command. -def list_versions(purls, output, file, command_name): +def get_versions_details(purls, output, file, unique, command_name): """ Return a list of dictionaries containing version-related data for each PURL in the `purls` input list. `check_versions_purl()` will print an error message to the console (also displayed in the JSON output) when necessary. """ - purl_versions = [] - for purl in purls: - purl_data = {} - purl_data["purl"] = purl - purl_data["versions"] = [] + versions_details = {} + versions_details["headers"] = [] + versions_details["packages"] = [] - purl = purl.strip() - if not purl: - continue + versions_warnings = {} - versions_purl = check_versions_purl(purl) + input_purls, normalized_purls = normalize_purls(purls, unique) - if command_name == "versions" and versions_purl == "not_valid": - print(f"'{purl}' not valid") - continue - - if command_name == "versions" and versions_purl == "valid_but_not_supported": - print(f"'{purl}' not supported with `versions` command") - continue + clear_log_file() - if command_name == "versions" and versions_purl == "not_in_upstream_repo": - print(f"'{purl}' does not exist in the upstream repo") + for purl in input_purls: + purl = purl.strip() + if not purl: continue - # TODO: Add to warnings and test it as well. - if command_name == "versions" and versions_purl == "validation_error": - print(f"'{purl}' encountered a validation error") - continue + purl_data = {} + purl_data["purl"] = purl + purl_data["versions"] = [] - # TODO: Is this needed to catch the intermittent fetchcode/package_versions.py versions()/get_response() `Error while fetching` error? I don't think so. - # if command_name == "versions" and versions_purl == "error_fetching_purl": - # print(f"Error fetching '{purl}'") + versions_purl = check_versions_purl(purl) - # TODO: Is the subsumed by the preceding `validation_error`? I think YES. - # if versions(purl) is None: - # print(f"{purl} encountered a versions(purl) error") - # continue + if command_name == "versions" and versions_purl: + versions_warnings[purl] = versions_purl + continue - for package_version_object in list(versions(purl)): + for package_version in list(versions(purl)): purl_version_data = {} - purl_version = package_version_object.to_dict()["value"] - nested_purl = purl + "@" + f"{purl_version}" + purl_version = package_version.to_dict()["value"] + + # We use `versions()` from fetchcode/package_versions.py, which + # keeps the version (if any) of the input PURL in its output, so + # "pkg:pypi/fetchcode@0.3.0" is returned as + # "pkg:pypi/fetchcode@0.3.0@0.1.0", "pkg:pypi/fetchcode@0.3.0@0.2.0" + # etc. Thus, we remove any string starting with `@` first. + raw_purl = purl = re.split("[@,]+", purl)[0] + nested_purl = raw_purl + "@" + f"{purl_version}" purl_version_data["purl"] = nested_purl purl_version_data["version"] = f"{purl_version}" purl_version_data["release_date"] = ( - f'{package_version_object.to_dict()["release_date"]}' + f'{package_version.to_dict()["release_date"]}' ) purl_data["versions"].append(purl_version_data) - purl_versions.append(purl_data) + versions_details["packages"].append(purl_data) - return purl_versions + versions_details["headers"] = construct_headers( + purls=purls, + output=output, + file=file, + command_name=command_name, + normalized_purls=normalized_purls, + unique=unique, + purl_warnings=versions_warnings, + ) + + return versions_details def check_versions_purl(purl): """ - Return a message for printing to the console if the input PURL is invalid, - its type is not supported by `versions` or its existence was not validated. + Return a variable identifying the message for printing to the console by + get_versions_details() if (1) the input PURL is invalid, (2) its type is not + supported by `versions` or (3) its existence was not validated (e.g., + "does not exist in the upstream repo"). + + This message will also be reported by construct_headers() in the + `warnings` field of the `header` section of the JSON object returned by + the `versions` command. Note for dev purposes: SUPPORTED_ECOSYSTEMS (imported from fetchcode.package_versions) comprises the following types: @@ -715,25 +801,15 @@ def check_versions_purl(purl): "pypi", ] """ - check_validation = validate_purls([purl]) + check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation[0] - - # TODO: Is this needed to catch the intermittent fetchcode/package_versions.py versions()/get_response() `Error while fetching` error? No, it does not catch that error. - # 2024-02-27 Tuesday 16:43:54. Just got one: - # (venv) Tue Feb 27, 2024 04:40 PM /home/jmh/dev/nexb/purldb jmh (247-purlcli-update-validate-and-versions) - # $ python -m purldb_toolkit.purlcli versions --purl pkg:gem/bundler-sass --purl pkg:deb/debian/2ping --output - - # Error while fetching 'https://sources.debian.org/api/src/2ping': 503 - # Traceback (most recent call last): - # if results is None: - # return "error_fetching_purl" + results = check_validation if results["valid"] == False: return "not_valid" supported = SUPPORTED_ECOSYSTEMS - versions_purl = PackageURL.from_string(purl) if versions_purl.type not in supported: @@ -757,5 +833,11 @@ def check_for_duplicate_input_sources(purls, file): raise click.UsageError("Use either purls or file.") +def clear_log_file(): + log_file = Path("purldb-toolkit/src/purldb_toolkit/app.log") + if log_file.is_file(): + os.remove(log_file) + + if __name__ == "__main__": purlcli() diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json new file mode 100644 index 00000000..928822f8 --- /dev/null +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json @@ -0,0 +1,81 @@ +{ + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "validate", + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:gem/rails", + "pkg:rubygems/rails" + ], + "--file": null, + "--output": "" + }, + "purls": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:gem/rails", + "pkg:rubygems/rails" + ], + "errors": [], + "warnings": [] + } + ], + "packages": [ + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.3.0" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.3.0?os=windows" + }, + { + "valid": true, + "exists": false, + "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.3.0os=windows" + }, + { + "valid": true, + "exists": false, + "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", + "purl": "pkg:pypi/fetchcode@5.0.0" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:cargo/banquo" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:gem/rails" + } + ] +} diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json new file mode 100644 index 00000000..bd3349d5 --- /dev/null +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json @@ -0,0 +1,63 @@ +{ + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "validate", + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:gem/rails", + "pkg:rubygems/rails" + ], + "--file": null, + "--unique": true, + "--output": "" + }, + "purls": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:gem/rails", + "pkg:rubygems/rails" + ], + "errors": [], + "warnings": [ + "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'" + ] + } + ], + "packages": [ + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:cargo/banquo" + }, + { + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:gem/rails" + } + ] +} diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json new file mode 100644 index 00000000..10443de7 --- /dev/null +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json @@ -0,0 +1,176 @@ +{ + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:hex/coherence@0.1.0" + ], + "--file": null, + "--output": "" + }, + "purls": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:hex/coherence@0.1.0" + ], + "errors": [], + "warnings": [ + "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", + "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", + "'pkg:nginx/nginx' not supported with `versions` command" + ] + } + ], + "packages": [ + { + "purl": "pkg:pypi/fetchcode", + "versions": [ + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25T15:15:15.265015+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14T16:36:02.242182+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18T20:49:45.840364+00:00" + } + ] + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "versions": [ + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25T15:15:15.265015+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14T16:36:02.242182+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18T20:49:45.840364+00:00" + } + ] + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", + "versions": [ + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25T15:15:15.265015+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14T16:36:02.242182+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18T20:49:45.840364+00:00" + } + ] + }, + { + "purl": "pkg:cargo/banquo", + "versions": [ + { + "purl": "pkg:cargo/banquo@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07T23:21:50.548891+00:00" + } + ] + }, + { + "purl": "pkg:hex/coherence@0.1.0", + "versions": [ + { + "purl": "pkg:hex/coherence@0.8.0", + "version": "0.8.0", + "release_date": "2023-09-22T18:28:36.224103+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.2", + "version": "0.5.2", + "release_date": "2018-09-03T23:52:38.161321+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.1", + "version": "0.5.1", + "release_date": "2018-08-28T01:33:14.565151+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.0", + "version": "0.5.0", + "release_date": "2017-08-02T06:23:12.948525+00:00" + }, + { + "purl": "pkg:hex/coherence@0.4.0", + "version": "0.4.0", + "release_date": "2017-07-03T21:55:56.591426+00:00" + }, + { + "purl": "pkg:hex/coherence@0.3.1", + "version": "0.3.1", + "release_date": "2016-11-27T05:30:34.553920+00:00" + }, + { + "purl": "pkg:hex/coherence@0.3.0", + "version": "0.3.0", + "release_date": "2016-08-28T19:04:10.794525+00:00" + }, + { + "purl": "pkg:hex/coherence@0.2.0", + "version": "0.2.0", + "release_date": "2016-07-30T21:07:45.377540+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.3", + "version": "0.1.3", + "release_date": "2016-07-19T03:33:09.185782+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.2", + "version": "0.1.2", + "release_date": "2016-07-12T18:41:27.084599+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.1", + "version": "0.1.1", + "release_date": "2016-07-11T13:56:26.388096+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.0", + "version": "0.1.0", + "release_date": "2016-07-11T06:52:43.545719+00:00" + } + ] + } + ] +} diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json new file mode 100644 index 00000000..0a7b0869 --- /dev/null +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json @@ -0,0 +1,140 @@ +{ + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:hex/coherence@0.1.0" + ], + "--file": null, + "--unique": true, + "--output": "" + }, + "purls": [ + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/fetchcode@5.0.0", + "pkg:cargo/banquo", + "pkg:nginx/nginx", + "pkg:hex/coherence@0.1.0" + ], + "errors": [], + "warnings": [ + "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", + "input PURL: 'pkg:hex/coherence@0.1.0' normalized to 'pkg:hex/coherence'", + "'pkg:nginx/nginx' not supported with `versions` command" + ] + } + ], + "packages": [ + { + "purl": "pkg:pypi/fetchcode", + "versions": [ + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25T15:15:15.265015+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14T16:36:02.242182+00:00" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18T20:49:45.840364+00:00" + } + ] + }, + { + "purl": "pkg:cargo/banquo", + "versions": [ + { + "purl": "pkg:cargo/banquo@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07T23:21:50.548891+00:00" + } + ] + }, + { + "purl": "pkg:hex/coherence", + "versions": [ + { + "purl": "pkg:hex/coherence@0.8.0", + "version": "0.8.0", + "release_date": "2023-09-22T18:28:36.224103+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.2", + "version": "0.5.2", + "release_date": "2018-09-03T23:52:38.161321+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.1", + "version": "0.5.1", + "release_date": "2018-08-28T01:33:14.565151+00:00" + }, + { + "purl": "pkg:hex/coherence@0.5.0", + "version": "0.5.0", + "release_date": "2017-08-02T06:23:12.948525+00:00" + }, + { + "purl": "pkg:hex/coherence@0.4.0", + "version": "0.4.0", + "release_date": "2017-07-03T21:55:56.591426+00:00" + }, + { + "purl": "pkg:hex/coherence@0.3.1", + "version": "0.3.1", + "release_date": "2016-11-27T05:30:34.553920+00:00" + }, + { + "purl": "pkg:hex/coherence@0.3.0", + "version": "0.3.0", + "release_date": "2016-08-28T19:04:10.794525+00:00" + }, + { + "purl": "pkg:hex/coherence@0.2.0", + "version": "0.2.0", + "release_date": "2016-07-30T21:07:45.377540+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.3", + "version": "0.1.3", + "release_date": "2016-07-19T03:33:09.185782+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.2", + "version": "0.1.2", + "release_date": "2016-07-12T18:41:27.084599+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.1", + "version": "0.1.1", + "release_date": "2016-07-11T13:56:26.388096+00:00" + }, + { + "purl": "pkg:hex/coherence@0.1.0", + "version": "0.1.0", + "release_date": "2016-07-11T06:52:43.545719+00:00" + } + ] + } + ] +} diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 6aa3b13f..ebc276e6 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -11,9 +11,7 @@ import os from collections import OrderedDict -import click import pytest -import requests from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting from purldb_toolkit import cli_test_utils, purlcli @@ -97,19 +95,14 @@ def test_metadata_cli(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - (output_data["packages"], expected_data["packages"]), ] for output, expected in result_objects: assert output == expected - """ - QUESTION: Is this a better way to test the contents of `packages`? - We already remove some dynamic fields like `download_url`, but - `metadata` also adds new versions as they appear. The below approach - avoids an error from a new version while checking whether the existing - expected versions still appear in the result data. - """ + # NOTE: To avoid errors from the addition of new versions, we exclude + # `(output_data["packages"], expected_data["packages"])` from the + # result_objects list above and handle here. for expected in expected_data["packages"]: assert expected in output_data["packages"] @@ -189,16 +182,12 @@ def test_metadata_cli_unique(self): output_data["headers"][0]["options"]["--unique"], expected_data["headers"][0]["options"]["--unique"], ), - (output_data["packages"], expected_data["packages"]), ] for output, expected in result_objects: assert output == expected - """ - QUESTION: Is this a better way to test the contents of `packages`? - See point under test_metadata_cli() re addition of new versions. - """ + # See note under test_metadata_cli() re addition of new versions. for expected in expected_data["packages"]: assert expected in output_data["packages"] @@ -606,12 +595,7 @@ def test_metadata_details(self, test_input, expected): cli_test_utils.streamline_headers(expected["headers"]) streamline_metadata_packages(expected["packages"]) - assert purl_metadata == expected - - """ - QUESTION: Is this a better way to test the contents of `packages`? - See note under test_metadata_cli() re addition of new versions. - """ + # See note under test_metadata_cli() re addition of new versions. assert purl_metadata["headers"] == expected["headers"] for expected in expected["packages"]: @@ -694,16 +678,61 @@ def test_check_metadata_purl(self, test_input, expected): (["pkg:pypi/"]), ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), ), + ( + [ + [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:pypi/fetchcode@5.0.0", + "pkg:pypi/dejacode", + "pkg:pypi/dejacode@5.0.0", + "pkg:pypi/dejacode@5.0.0?os=windows", + "pkg:pypi/dejacode@5.0.0os=windows", + "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", + "pkg:pypi/dejacode@5.0.0#how/are/you", + "pkg:pypi/dejacode@10.0.0", + "pkg:cargo/banquo", + "pkg:cargo/socksprox", + "pkg:nginx/nginx", + "pkg:nginx/nginx@0.8.9?os=windows", + ] + ], + ( + [ + "pkg:pypi/fetchcode", + "pkg:pypi/dejacode", + "pkg:cargo/banquo", + "pkg:cargo/socksprox", + "pkg:nginx/nginx", + ] + ), + ( + [ + ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), + ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), + ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), + ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), + ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), + ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), + ( + "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", + "pkg:pypi/dejacode", + ), + ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), + ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), + ("pkg:cargo/banquo", "pkg:cargo/banquo"), + ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), + ("pkg:nginx/nginx", "pkg:nginx/nginx"), + ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), + ] + ), + ), ], ) def test_normalize_purls( self, test_input, expected_input_purls, expected_normalized_purls ): - input_purls = [] - normalized_purls = [] - input_purls, normalized_purls = purlcli.normalize_purls( - test_input[0], input_purls, normalized_purls - ) + unique = True + input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) assert input_purls == expected_input_purls assert normalized_purls == expected_normalized_purls @@ -833,10 +862,6 @@ class TestPURLCLI_urls(object): def test_urls_cli(self): """ Test the `urls` command with actual and expected JSON output files. - - Note that we can't simply compare the actual and expected JSON files - because the `--output` values (paths) differ due to the use of - temporary files, and therefore we test a list of relevant key-value pairs. """ expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output.json" @@ -933,10 +958,6 @@ def test_urls_cli(self): def test_urls_cli_unique(self): """ Test the `urls` command with actual and expected JSON output files. - - Note that we can't simply compare the actual and expected JSON files - because the `--output` values (paths) differ due to the use of - temporary files, and therefore we test a list of relevant key-value pairs. """ expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_unique.json" @@ -1034,10 +1055,6 @@ def test_urls_cli_unique(self): def test_urls_cli_head(self): """ Test the `urls` command with actual and expected JSON output files. - - Note that we can't simply compare the actual and expected JSON files - because the `--output` values (paths) differ due to the use of - temporary files, and therefore we test a list of relevant key-value pairs. """ expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head.json" @@ -1504,261 +1521,687 @@ def test_make_head_request(self, test_input, expected): assert purl_status_code == expected -# TODO: not yet converted to a SCTK-like data structure. class TestPURLCLI_validate(object): - @pytest.mark.parametrize( - "test_input,expected", - [ + def test_validate_cli(self): + """ + Test the `validate` command with actual and expected JSON output files. + """ + expected_result_file = test_env.get_test_loc( + "purlcli/expected_validate_output.json" + ) + actual_result_file = test_env.get_temp_file("actual_validate_output.json") + options = [ + "--purl", + "pkg:pypi/fetchcode", + "--purl", + "pkg:pypi/fetchcode@0.3.0", + "--purl", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "--purl", + "pkg:pypi/fetchcode@0.3.0os=windows", + "--purl", + "pkg:pypi/fetchcode@5.0.0", + "--purl", + "pkg:cargo/banquo", + "--purl", + "pkg:nginx/nginx", + "--purl", + "pkg:gem/rails", + "--purl", + "pkg:rubygems/rails", + "--output", + actual_result_file, + ] + runner = CliRunner() + result = runner.invoke(purlcli.validate, options, catch_exceptions=False) + assert result.exit_code == 0 + + f_output = open(actual_result_file) + output_data = json.load(f_output) + + f_expected = open(expected_result_file) + expected_data = json.load(f_expected) + + result_objects = [ ( - ["pkg:pypi/fetchcode@0.2.0"], - [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode@0.2.0", - } - ], + output_data["headers"][0]["tool_name"], + expected_data["headers"][0]["tool_name"], ), + (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( - ["pkg:pypi/fetchcode@10.2.0"], - [ - { - "valid": True, - "exists": False, - "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", - "purl": "pkg:pypi/fetchcode@10.2.0", - } - ], + output_data["headers"][0]["warnings"], + expected_data["headers"][0]["warnings"], ), ( - ["pkg:nginx/nginx@0.8.9?os=windows"], - [ - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx@0.8.9?os=windows", - } - ], + output_data["headers"][0]["errors"], + expected_data["headers"][0]["errors"], ), ( - ["pkg:gem/bundler-sass"], - [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/bundler-sass", - } - ], + output_data["headers"][0]["options"]["command"], + expected_data["headers"][0]["options"]["command"], ), ( - ["pkg:rubygems/bundler-sass"], - [ - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - } - ], + output_data["headers"][0]["options"]["--purl"], + expected_data["headers"][0]["options"]["--purl"], ), ( - ["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - } - ], + output_data["headers"][0]["options"]["--file"], + expected_data["headers"][0]["options"]["--file"], ), - ], - ) - def test_validate_purl(self, test_input, expected): - validated_purls = purlcli.validate_purls(test_input) - assert validated_purls == expected + (output_data["packages"], expected_data["packages"]), + ] - def test_validate_purl_empty(self): - test_purls = [] - validated_purls = purlcli.validate_purls(test_purls) - expected_results = [] - assert validated_purls == expected_results + for output, expected in result_objects: + assert output == expected - @pytest.mark.parametrize( - "test_input,expected", - [ + def test_validate_cli_unique(self): + """ + Test the `validate` command with actual and expected JSON output files + with the `--unique` flag included in the command. + """ + expected_result_file = test_env.get_test_loc( + "purlcli/expected_validate_output_unique.json" + ) + actual_result_file = test_env.get_temp_file("actual_validate_output.json") + options = [ + "--purl", + "pkg:pypi/fetchcode", + "--purl", + "pkg:pypi/fetchcode@0.3.0", + "--purl", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "--purl", + "pkg:pypi/fetchcode@0.3.0os=windows", + "--purl", + "pkg:pypi/fetchcode@5.0.0", + "--purl", + "pkg:cargo/banquo", + "--purl", + "pkg:nginx/nginx", + "--purl", + "pkg:gem/rails", + "--purl", + "pkg:rubygems/rails", + "--output", + actual_result_file, + "--unique", + ] + runner = CliRunner() + result = runner.invoke(purlcli.validate, options, catch_exceptions=False) + assert result.exit_code == 0 + + f_output = open(actual_result_file) + output_data = json.load(f_output) + + f_expected = open(expected_result_file) + expected_data = json.load(f_expected) + + result_objects = [ ( - ["pkg:pypi/fetchcode@0.2.0"], - [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode@0.2.0", - } - ], + output_data["headers"][0]["tool_name"], + expected_data["headers"][0]["tool_name"], ), + (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( - ["pkg:pypi/fetchcode@0.2.0?"], - [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode@0.2.0?", - } - ], + output_data["headers"][0]["warnings"], + expected_data["headers"][0]["warnings"], ), ( - ["pkg:pypi/fetchcode@?0.2.0"], - [ - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "pkg:pypi/fetchcode@?0.2.0", - } - ], + output_data["headers"][0]["errors"], + expected_data["headers"][0]["errors"], ), ( - ["foo"], - [ - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "foo", - } - ], + output_data["headers"][0]["options"]["command"], + expected_data["headers"][0]["options"]["command"], ), - ], - ) - def test_validate_purl_invalid(self, test_input, expected): - validated_purls = purlcli.validate_purls(test_input) - assert validated_purls == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ ( - ["pkg:nginx/nginx@0.8.9?os=windows"], - [ - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx@0.8.9?os=windows", - }, - ], + output_data["headers"][0]["options"]["--purl"], + expected_data["headers"][0]["options"]["--purl"], ), ( - [" pkg:nginx/nginx@0.8.9?os=windows"], - [ - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx@0.8.9?os=windows", - }, - ], + output_data["headers"][0]["options"]["--file"], + expected_data["headers"][0]["options"]["--file"], ), ( - ["pkg:nginx/nginx@0.8.9?os=windows "], - [ - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx@0.8.9?os=windows", - } - ], + output_data["headers"][0]["options"]["--unique"], + expected_data["headers"][0]["options"]["--unique"], ), - ], - ) - def test_validate_purl_strip(self, test_input, expected): - validated_purls = purlcli.validate_purls(test_input) - assert validated_purls == expected + (output_data["packages"], expected_data["packages"]), + ] + for output, expected in result_objects: + assert output == expected -# TODO: not yet converted to a SCTK-like data structure. -class TestPURLCLI_versions(object): @pytest.mark.parametrize( "test_input,expected", [ ( - ["pkg:pypi/fetchcode"], - [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25T15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14T16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18T20:49:45.840364+00:00", - }, - ], - }, - ], - ), - ( - ["pkg:gem/bundler-sass"], - [ - { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "release_date": "2013-12-11T00:27:10.097000+00:00", - "version": "0.1.2", - }, - ], - }, - ], + "pkg:pypi/fetchcode@0.2.0", + { + "valid": True, + "exists": True, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.2.0", + }, ), ( - ["pkg:rubygems/bundler-sass"], - [], + "pkg:pypi/fetchcode@10.2.0", + { + "valid": True, + "exists": False, + "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", + "purl": "pkg:pypi/fetchcode@10.2.0", + }, ), ( - ["pkg:nginx/nginx"], - [], + "pkg:nginx/nginx@0.8.9?os=windows", + { + "valid": True, + "exists": None, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", + "purl": "pkg:nginx/nginx@0.8.9?os=windows", + }, ), ( - ["pkg:pypi/zzzzz"], - [], + "pkg:gem/bundler-sass", + { + "valid": True, + "exists": True, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:gem/bundler-sass", + }, ), ( - ["pkg:pypi/?fetchcode"], - [], + "pkg:rubygems/bundler-sass", + { + "valid": True, + "exists": None, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", + "purl": "pkg:rubygems/bundler-sass", + }, ), ( - ["zzzzz"], - [], + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + { + "valid": True, + "exists": True, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + }, ), ], ) - def test_versions(self, test_input, expected): - # TODO: not yet updated to SCTK-like structure. - output = "" + def test_validate_purl(self, test_input, expected): + validated_purl = purlcli.validate_purl(test_input) + assert validated_purl == expected + + def test_validate_purl_empty(self): + test_input = None + validated_purl = purlcli.validate_purl(test_input) + expected_results = {"errors": {"purl": ["This field is required."]}} + assert validated_purl == expected_results + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + "pkg:pypi/fetchcode@0.2.0", + { + "valid": True, + "exists": True, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.2.0", + }, + ), + ( + "pkg:pypi/fetchcode@0.2.0?", + { + "valid": True, + "exists": True, + "message": "The provided Package URL is valid, and the package exists in the upstream repo.", + "purl": "pkg:pypi/fetchcode@0.2.0?", + }, + ), + ( + "pkg:pypi/fetchcode@?0.2.0", + { + "valid": False, + "exists": None, + "message": "The provided PackageURL is not valid.", + "purl": "pkg:pypi/fetchcode@?0.2.0", + }, + ), + ( + "foo", + { + "valid": False, + "exists": None, + "message": "The provided PackageURL is not valid.", + "purl": "foo", + }, + ), + ], + ) + def test_validate_purl_invalid(self, test_input, expected): + validated_purl = purlcli.validate_purl(test_input) + assert validated_purl == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + "pkg:nginx/nginx@0.8.9?os=windows", + { + "valid": True, + "exists": None, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", + "purl": "pkg:nginx/nginx@0.8.9?os=windows", + }, + ), + ( + " pkg:nginx/nginx@0.8.9?os=windows", + { + "valid": True, + "exists": None, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", + "purl": "pkg:nginx/nginx@0.8.9?os=windows", + }, + ), + ( + "pkg:nginx/nginx@0.8.9?os=windows ", + { + "valid": True, + "exists": None, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", + "purl": "pkg:nginx/nginx@0.8.9?os=windows", + }, + ), + ], + ) + def test_validate_purl_strip(self, test_input, expected): + validated_purl = purlcli.validate_purl(test_input) + assert validated_purl == expected + + +class TestPURLCLI_versions(object): + def test_versions_cli(self): + """ + Test the `versions` command with actual and expected JSON output files. + """ + expected_result_file = test_env.get_test_loc( + "purlcli/expected_versions_output.json" + ) + actual_result_file = test_env.get_temp_file("actual_versions_output.json") + options = [ + "--purl", + "pkg:pypi/fetchcode", + "--purl", + "pkg:pypi/fetchcode@0.3.0", + "--purl", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "--purl", + "pkg:pypi/fetchcode@0.3.0os=windows", + "--purl", + "pkg:pypi/fetchcode@5.0.0", + "--purl", + "pkg:cargo/banquo", + "--purl", + "pkg:nginx/nginx", + "--purl", + "pkg:hex/coherence@0.1.0", + "--output", + actual_result_file, + ] + runner = CliRunner() + result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) + assert result.exit_code == 0 + + f_output = open(actual_result_file) + output_data = json.load(f_output) + cli_test_utils.streamline_headers(output_data["headers"]) + + f_expected = open(expected_result_file) + expected_data = json.load(f_expected) + cli_test_utils.streamline_headers(expected_data["headers"]) + + result_objects = [ + ( + output_data["headers"][0]["tool_name"], + expected_data["headers"][0]["tool_name"], + ), + (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + ( + output_data["headers"][0]["warnings"], + expected_data["headers"][0]["warnings"], + ), + ( + output_data["headers"][0]["errors"], + expected_data["headers"][0]["errors"], + ), + ( + output_data["headers"][0]["options"]["command"], + expected_data["headers"][0]["options"]["command"], + ), + ( + output_data["headers"][0]["options"]["--purl"], + expected_data["headers"][0]["options"]["--purl"], + ), + ( + output_data["headers"][0]["options"]["--file"], + expected_data["headers"][0]["options"]["--file"], + ), + ] + + for output, expected in result_objects: + assert output == expected + + # NOTE: To avoid errors from the addition of new versions, we exclude + # `(output_data["packages"], expected_data["packages"])` from the + # result_objects list above and handle here. + expected_versions = [] + output_versions = [] + for expected in expected_data["packages"]: + expected_versions = expected["versions"] + for output in output_data["packages"]: + output_versions = output["versions"] + + assert [i for i in expected_versions if i not in output_versions] == [] + + def test_versions_cli_unique(self): + """ + Test the `versions` command with actual and expected JSON output files + with the `--unique` flag included in the command. + """ + expected_result_file = test_env.get_test_loc( + "purlcli/expected_versions_output_unique.json" + ) + actual_result_file = test_env.get_temp_file("actual_versions_output.json") + options = [ + "--purl", + "pkg:pypi/fetchcode", + "--purl", + "pkg:pypi/fetchcode@0.3.0", + "--purl", + "pkg:pypi/fetchcode@0.3.0?os=windows", + "--purl", + "pkg:pypi/fetchcode@0.3.0os=windows", + "--purl", + "pkg:pypi/fetchcode@5.0.0", + "--purl", + "pkg:cargo/banquo", + "--purl", + "pkg:nginx/nginx", + "--purl", + "pkg:hex/coherence@0.1.0", + "--output", + actual_result_file, + "--unique", + ] + runner = CliRunner() + result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) + assert result.exit_code == 0 + + f_output = open(actual_result_file) + output_data = json.load(f_output) + cli_test_utils.streamline_headers(output_data["headers"]) + + f_expected = open(expected_result_file) + expected_data = json.load(f_expected) + cli_test_utils.streamline_headers(expected_data["headers"]) + + result_objects = [ + ( + output_data["headers"][0]["tool_name"], + expected_data["headers"][0]["tool_name"], + ), + (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + ( + output_data["headers"][0]["warnings"], + expected_data["headers"][0]["warnings"], + ), + ( + output_data["headers"][0]["errors"], + expected_data["headers"][0]["errors"], + ), + ( + output_data["headers"][0]["options"]["command"], + expected_data["headers"][0]["options"]["command"], + ), + ( + output_data["headers"][0]["options"]["--purl"], + expected_data["headers"][0]["options"]["--purl"], + ), + ( + output_data["headers"][0]["options"]["--file"], + expected_data["headers"][0]["options"]["--file"], + ), + ( + output_data["headers"][0]["options"]["--unique"], + expected_data["headers"][0]["options"]["--unique"], + ), + ] + + for output, expected in result_objects: + assert output == expected + + # See note under test_versions_cli() re addition of new versions. + expected_versions = [] + output_versions = [] + for expected in expected_data["packages"]: + expected_versions = expected["versions"] + for output in output_data["packages"]: + output_versions = output["versions"] + + assert [i for i in expected_versions if i not in output_versions] == [] + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + ["pkg:pypi/fetchcode"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:pypi/fetchcode"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:pypi/fetchcode"], + "errors": [], + "warnings": [], + } + ], + "packages": [ + { + "purl": "pkg:pypi/fetchcode", + "versions": [ + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25T15:15:15.265015+00:00", + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14T16:36:02.242182+00:00", + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18T20:49:45.840364+00:00", + }, + ], + } + ], + }, + ), + ( + ["pkg:gem/bundler-sass"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:gem/bundler-sass"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:gem/bundler-sass"], + "errors": [], + "warnings": [], + } + ], + "packages": [ + { + "purl": "pkg:gem/bundler-sass", + "versions": [ + { + "purl": "pkg:gem/bundler-sass@0.1.2", + "version": "0.1.2", + "release_date": "2013-12-11T00:27:10.097000+00:00", + } + ], + } + ], + }, + ), + ( + ["pkg:rubygems/bundler-sass"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:rubygems/bundler-sass"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:rubygems/bundler-sass"], + "errors": [], + "warnings": [ + "'pkg:rubygems/bundler-sass' not supported with `versions` command" + ], + } + ], + "packages": [], + }, + ), + ( + ["pkg:nginx/nginx"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:nginx/nginx"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:nginx/nginx"], + "errors": [], + "warnings": [ + "'pkg:nginx/nginx' not supported with `versions` command" + ], + } + ], + "packages": [], + }, + ), + ( + ["pkg:pypi/zzzzz"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:pypi/zzzzz"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:pypi/zzzzz"], + "errors": [], + "warnings": [ + "'pkg:pypi/zzzzz' does not exist in the upstream repo" + ], + } + ], + "packages": [], + }, + ), + ( + ["pkg:pypi/?fetchcode"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["pkg:pypi/?fetchcode"], + "--file": None, + "--output": "", + }, + "purls": ["pkg:pypi/?fetchcode"], + "errors": [], + "warnings": ["'pkg:pypi/?fetchcode' not valid"], + } + ], + "packages": [], + }, + ), + ( + ["zzzzz"], + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": ["zzzzz"], + "--file": None, + "--output": "", + }, + "purls": ["zzzzz"], + "errors": [], + "warnings": ["'zzzzz' not valid"], + } + ], + "packages": [], + }, + ), + ], + ) + def test_versions_details(self, test_input, expected): + output = "" file = "" command_name = "versions" + unique = False - purl_versions = purlcli.list_versions(test_input, output, file, command_name) - # TODO: consider `expected in purl_versions` instead of `purl_versions == expected` ==> handles dynamic data in the result better. - assert purl_versions == expected + purl_versions = purlcli.get_versions_details( + test_input, output, file, unique, command_name + ) + + cli_test_utils.streamline_headers(purl_versions["headers"]) + cli_test_utils.streamline_headers(expected["headers"]) + + # See note under test_versions_cli() re addition of new versions. + assert purl_versions["headers"] == expected["headers"] + + for expected in expected["packages"]: + assert expected in purl_versions["packages"] @pytest.mark.parametrize( "test_input,expected",