diff --git a/packagedb/tests/test_purlcli.py b/packagedb/tests/test_purlcli.py index cd3c7c83..85bc678f 100644 --- a/packagedb/tests/test_purlcli.py +++ b/packagedb/tests/test_purlcli.py @@ -1,14 +1,20 @@ +import json import os from click.testing import CliRunner from commoncode.testcase import FileBasedTesting +from django.test import TestCase +from fetchcode.package_versions import PackageVersion, router, versions import purlcli +from packagedb.models import Package class TestPURLCLI(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), "data") + # QUESTION: These four validate tests call validate_purls(), which queries the validate endpoint. Is that what we want here? I think we might be able to mock the API if we pass the API to the validate_purls() function instead of defining the API inside the function as we do now. + def test_validate_purl(self): test_purls = [ "pkg:nginx/nginx@0.8.9?os=windows", @@ -92,3 +98,92 @@ def test_validate_purl_strip(self): ] self.assertEqual(validated_purls, expected_results) + + def test_versions(self): + purls = ["pkg:pypi/fetchcode"] + abc = purlcli.list_versions(purls) + + print(f"\nabc = {abc}") + + for purl_list in abc: + for p in purl_list: + # print(PackageVersion.to_dict(p)) + p_dict = PackageVersion.to_dict(p) + # print(f"\np_dict = {p_dict}") + # print(f"\ntype(p_dict) = {type(p_dict)}") + + json_p_dict = json.dumps(p_dict) + # print(f"\njson_p_dict = {json_p_dict}") + # print(f"\ntype(json_p_dict) = {type(json_p_dict)}") + + +# 2024-01-08 Monday 17:55:15. Based on test_api.py's class PurlValidateApiTestCase(TestCase). +class TestPURLCLI_API(TestCase): + def setUp(self): + self.package_data = { + "type": "npm", + "namespace": "", + "name": "foobar", + "version": "1,1.0", + "qualifiers": "", + "subpath": "", + "download_url": "", + "filename": "Foo.zip", + "sha1": "testsha1", + "md5": "testmd5", + "size": 101, + } + self.package = Package.objects.create(**self.package_data) + self.package.refresh_from_db() + + def test_api_purl_validation(self): + data1 = { + "purl": "pkg:npm/foobar@1.1.0", + "check_existence": True, + } + response1 = self.client.get(f"/api/validate/", data=data1) + + print(f"\nresponse1 = {response1}") + print(f"\nresponse1.data = {response1.data}") + print(f"") + + data2 = { + "purl": "pkg:npm/?foobar@1.1.0", + "check_existence": True, + } + response2 = self.client.get(f"/api/validate/", data=data2) + + print(f"\nresponse2 = {response2}") + print(f"\nresponse2.data = {response2.data}") + print(f"") + + self.assertEquals(True, response1.data["valid"]) + self.assertEquals(True, response1.data["exists"]) + self.assertEquals( + "The provided Package URL is valid, and the package exists in the upstream repo.", + response1.data["message"], + ) + + self.assertEquals(False, response2.data["valid"]) + self.assertEquals( + "The provided PackageURL is not valid.", response2.data["message"] + ) + + # ZZZ: 2024-01-08 Monday 18:54:51. Some exploring: + + data3 = { + "purl": "pkg:npm/ogdendunes", + "check_existence": True, + } + response3 = self.client.get(f"/api/validate/", data=data3) + + print(f"\nresponse3 = {response3}") + print(f"\nresponse3.data = {response3.data}") + print(f"") + + self.assertEqual(True, response3.data["valid"]) + self.assertEqual(False, response3.data["exists"]) + self.assertEqual( + "The provided Package URL is valid but does not exist in the upstream repo.", + response3.data["message"], + ) diff --git a/purlcli.py b/purlcli.py index 013170df..44be1d19 100644 --- a/purlcli.py +++ b/purlcli.py @@ -2,6 +2,7 @@ import click import requests +from fetchcode.package_versions import PackageVersion, router, versions @click.group() @@ -42,13 +43,17 @@ def validate(purls, output, file): if file: purls = file.read().splitlines(False) - validated_purls = validate_purls(purls) + api_query = "https://public.purldb.io/api/validate/" + + # validated_purls = validate_purls(purls) + validated_purls = validate_purls(purls, api_query) json.dump(validated_purls, output, indent=4) -def validate_purls(purls): - api_query = "https://public.purldb.io/api/validate/" +# def validate_purls(purls): +def validate_purls(purls, api_query): + # api_query = "https://public.purldb.io/api/validate/" validated_purls = [] for purl in purls: purl = purl.strip() @@ -62,5 +67,173 @@ def validate_purls(purls): return validated_purls +@purlcli.command(name="versions") +@click.option( + "--purl", + "purls", + multiple=True, + required=False, + help="PackageURL or PURL.", +) +@click.option( + "--output", + type=click.File(mode="w", encoding="utf-8"), + required=True, + default="-", + help="Write validation output as JSON to FILE.", +) +@click.option( + "--file", + type=click.File(mode="r", encoding="utf-8"), + required=False, + help="Read a list of PURLs from a FILE, one per line.", +) +def get_versions(purls, output, file): + """ + Check the syntax of one or more PURLs. + """ + if (purls and file) or not (purls or file): + raise click.UsageError("Use either purls or file but not both.") + + if file: + purls = file.read().splitlines(False) + + purl_versions = list_versions(purls) + + purl_versions + # print(f"\nlen(purl_versions) = {len(purl_versions)}") + + +def list_versions(purls): + print(f"\nlen(purls) = {len(purls)}") + print(f"\ntype(purls) = {type(purls)}") + print(f"\npurls = {purls}") + purl_versions = [] + + list_of_dict_outputs = [] + # and this will hold the dict_outputs converted with json.dumps(): + list_of_dict_output_json_dumps = [] + for purl in purls: + dict_output = {} + print(f"\n==> purl = {purl}") + dict_output["purl"] = purl + dict_output["versions"] = [] + purl = purl.strip() + if not purl: + continue + + # This works: this is a list of PackageVersion objects + results = list(router.process(purl)) + + print(f"\n\nrouter.process(purl) = {router.process(purl)}") + + print(f"\nlist(router.process(purl)) = {list(router.process(purl))}") + + # versions(purl) is a generator object + print(f"\nversions(purl) = {versions(purl)}") + + # Another test -- this is a list of PackageVersion objects + results_versions = list(versions(purl)) + print(f"\nresults_versions = {results_versions}") + + purl_versions.append(results) + + # Test: list of strings + results_values = [v.value for v in router.process(purl)] + print(f"\nresults_values = {results_values}") + + # 2024-01-05 Friday 17:25:41. Iterate through PackageVersion() objects + for package_version_object in list(versions(purl)): + print(f"\n*** package_version_object = {package_version_object}") + + print( + f"\n*** package_version_object.to_dict() = {package_version_object.to_dict()}" + ) + + print( + f"\n*** package_version_object.to_dict()['value'] = {package_version_object.to_dict()['value']}" + ) + + print( + f"\n*** package_version_object.to_dict()['release_date'] = {package_version_object.to_dict()['release_date']}" + ) + + # Here, too, create dict which we'll convert to JSON with json.dumps(). + nested_dict = {} + print(f"type(nested_dict) = {type(nested_dict)}") + + nested_purl = purl + "@" + f'{package_version_object.to_dict()["value"]}' + # nested_purl = "TEST" + + # dict_output["versions"].append({"purl": nested_purl}) + nested_dict["purl"] = nested_purl + nested_dict["version"] = f'{package_version_object.to_dict()["value"]}' + nested_dict[ + "release_date" + ] = f'{package_version_object.to_dict()["release_date"]}' + dict_output["versions"].append(nested_dict) + + # dict + print(f"\n==> dict_output = {dict_output}") + print(f"\n==> type(dict_output) = {type(dict_output)}") + # add to list + list_of_dict_outputs.append(dict_output) + # 2024-01-05 Friday 20:53:18. Does this format? + # list_of_dict_outputs.append(json.dumps(dict_output, indent=2, sort_keys=False)) + + # 2024-01-05 Friday 18:31:54. Let's convert dict_output with json.loads() and json.dumps() -- but that errors. Omit json.loads()??? + # json.dumps() works but the .json file is not formatted, i.e., no indents + dict_output_json_dumps = json.dumps(dict_output, indent=2, sort_keys=False) + + # no the next 2 fail. + # dict_output_json_loads = json.loads(dict_output) + # dict_output_json_dumps = json.dumps( + # dict_output_json_loads, indent=2, sort_keys=False + # ) + + print(f"\n==> dict_output_json_dumps = {dict_output_json_dumps}") + # and add to the separate list + list_of_dict_output_json_dumps.append(dict_output_json_dumps) + + print(f"\npurl_versions = {purl_versions}") + + print(f"\n==> list_of_dict_outputs = {list_of_dict_outputs}") + print(f"\n==> list_of_dict_output_json_dumps = {list_of_dict_output_json_dumps}") + + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_outputs-2024-01-08.json", + "w", + ) as f: + json.dump(list_of_dict_outputs, f) + + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_output_json_dumps-2024-01-08.json", + "w", + ) as f: + json.dump(list_of_dict_output_json_dumps, f) + + # try just one json.dumps() object -- dict_output_json_dumps -- NO: this also looks like this: + # "{\n \"purl\": \"pkg:pypi/minecode\",\n \"versions\": [\n {\n \"purl\": + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-single_dict_output_json_dumps-2024-01-08.json", + "w", + ) as f: + # json.dump(dict_output_json_dumps, f) + # json.dump(dict_output_json_dumps, f, indent=4) + # json.dump(list_of_dict_outputs, f) + + # 2024-01-05 Friday 20:54:27. This creates formatted JSON in the output file -- not in a list + # json.dump(dict_output, f, indent=4) + + # json.dump(list_of_dict_outputs, f, indent=4) + + # json.dump([obj for obj in list_of_dict_outputs], f) + + # 2024-01-05 Friday 21:11:21. THIS NOW WORKS! OUTPUT IS A LIST OF FORMATTED JSON OBJECTS! + json.dump([obj for obj in list_of_dict_outputs], f, indent=4) + + return purl_versions + + if __name__ == "__main__": purlcli() diff --git a/purlcli_SAVED.py b/purlcli_SAVED.py new file mode 100644 index 00000000..50bedfa4 --- /dev/null +++ b/purlcli_SAVED.py @@ -0,0 +1,239 @@ +import json + +import click +import requests +from fetchcode.package_versions import PackageVersion, router, versions + + +@click.group() +def purlcli(): + """ + Return information from a PURL. + """ + + +@purlcli.command(name="validate") +@click.option( + "--purl", + "purls", + multiple=True, + required=False, + help="PackageURL or PURL.", +) +@click.option( + "--output", + type=click.File(mode="w", encoding="utf-8"), + required=True, + default="-", + help="Write validation output as JSON to FILE.", +) +@click.option( + "--file", + type=click.File(mode="r", encoding="utf-8"), + required=False, + help="Read a list of PURLs from a FILE, one per line.", +) +def validate(purls, output, file): + """ + Check the syntax of one or more PURLs. + """ + if (purls and file) or not (purls or file): + raise click.UsageError("Use either purls or file but not both.") + + if file: + purls = file.read().splitlines(False) + + # api_query = "https://public.purldb.io/api/validate/" + + validated_purls = validate_purls(purls) + # validated_purls = validate_purls(purls, api_query) + + json.dump(validated_purls, output, indent=4) + + +def validate_purls(purls): + # def validate_purls(purls, api_query): + api_query = "https://public.purldb.io/api/validate/" + validated_purls = [] + for purl in purls: + purl = purl.strip() + if not purl: + continue + request_body = {"purl": purl, "check_existence": True} + response = requests.get(api_query, params=request_body) + results = response.json() + validated_purls.append(results) + + return validated_purls + + +@purlcli.command(name="versions") +@click.option( + "--purl", + "purls", + multiple=True, + required=False, + help="PackageURL or PURL.", +) +@click.option( + "--output", + type=click.File(mode="w", encoding="utf-8"), + required=True, + default="-", + help="Write validation output as JSON to FILE.", +) +@click.option( + "--file", + type=click.File(mode="r", encoding="utf-8"), + required=False, + help="Read a list of PURLs from a FILE, one per line.", +) +def get_versions(purls, output, file): + """ + Check the syntax of one or more PURLs. + """ + if (purls and file) or not (purls or file): + raise click.UsageError("Use either purls or file but not both.") + + if file: + purls = file.read().splitlines(False) + + purl_versions = list_versions(purls) + + purl_versions + # print(f"\nlen(purl_versions) = {len(purl_versions)}") + + +def list_versions(purls): + print(f"\nlen(purls) = {len(purls)}") + print(f"\ntype(purls) = {type(purls)}") + print(f"\npurls = {purls}") + purl_versions = [] + + list_of_dict_outputs = [] + # and this will hold the dict_outputs converted with json.dumps(): + list_of_dict_output_json_dumps = [] + for purl in purls: + dict_output = {} + print(f"\n==> purl = {purl}") + dict_output["purl"] = purl + dict_output["versions"] = [] + purl = purl.strip() + if not purl: + continue + + # This works: this is a list of PackageVersion objects + results = list(router.process(purl)) + + print(f"\n\nrouter.process(purl) = {router.process(purl)}") + + print(f"\nlist(router.process(purl)) = {list(router.process(purl))}") + + # versions(purl) is a generator object + print(f"\nversions(purl) = {versions(purl)}") + + # Another test -- this is a list of PackageVersion objects + results_versions = list(versions(purl)) + print(f"\nresults_versions = {results_versions}") + + purl_versions.append(results) + + # Test: list of strings + results_values = [v.value for v in router.process(purl)] + print(f"\nresults_values = {results_values}") + + # 2024-01-05 Friday 17:25:41. Iterate through PackageVersion() objects + for package_version_object in list(versions(purl)): + print(f"\n*** package_version_object = {package_version_object}") + + print( + f"\n*** package_version_object.to_dict() = {package_version_object.to_dict()}" + ) + + print( + f"\n*** package_version_object.to_dict()['value'] = {package_version_object.to_dict()['value']}" + ) + + print( + f"\n*** package_version_object.to_dict()['release_date'] = {package_version_object.to_dict()['release_date']}" + ) + + # Here, too, create dict which we'll convert to JSON with json.dumps(). + nested_dict = {} + print(f"type(nested_dict) = {type(nested_dict)}") + + nested_purl = purl + "@" + f'{package_version_object.to_dict()["value"]}' + # nested_purl = "TEST" + + # dict_output["versions"].append({"purl": nested_purl}) + nested_dict["purl"] = nested_purl + nested_dict["version"] = f'{package_version_object.to_dict()["value"]}' + nested_dict[ + "release_date" + ] = f'{package_version_object.to_dict()["release_date"]}' + dict_output["versions"].append(nested_dict) + + # dict + print(f"\n==> dict_output = {dict_output}") + print(f"\n==> type(dict_output) = {type(dict_output)}") + # add to list + list_of_dict_outputs.append(dict_output) + # 2024-01-05 Friday 20:53:18. Does this format? + # list_of_dict_outputs.append(json.dumps(dict_output, indent=2, sort_keys=False)) + + # 2024-01-05 Friday 18:31:54. Let's convert dict_output with json.loads() and json.dumps() -- but that errors. Omit json.loads()??? + # json.dumps() works but the .json file is not formatted, i.e., no indents + dict_output_json_dumps = json.dumps(dict_output, indent=2, sort_keys=False) + + # no the next 2 fail. + # dict_output_json_loads = json.loads(dict_output) + # dict_output_json_dumps = json.dumps( + # dict_output_json_loads, indent=2, sort_keys=False + # ) + + print(f"\n==> dict_output_json_dumps = {dict_output_json_dumps}") + # and add to the separate list + list_of_dict_output_json_dumps.append(dict_output_json_dumps) + + print(f"\npurl_versions = {purl_versions}") + + print(f"\n==> list_of_dict_outputs = {list_of_dict_outputs}") + print(f"\n==> list_of_dict_output_json_dumps = {list_of_dict_output_json_dumps}") + + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_outputs-2024-01-08.json", + "w", + ) as f: + json.dump(list_of_dict_outputs, f) + + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_output_json_dumps-2024-01-08.json", + "w", + ) as f: + json.dump(list_of_dict_output_json_dumps, f) + + # try just one json.dumps() object -- dict_output_json_dumps -- NO: this also looks like this: + # "{\n \"purl\": \"pkg:pypi/minecode\",\n \"versions\": [\n {\n \"purl\": + with open( + "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-single_dict_output_json_dumps-2024-01-08.json", + "w", + ) as f: + # json.dump(dict_output_json_dumps, f) + # json.dump(dict_output_json_dumps, f, indent=4) + # json.dump(list_of_dict_outputs, f) + + # 2024-01-05 Friday 20:54:27. This creates formatted JSON in the output file -- not in a list + # json.dump(dict_output, f, indent=4) + + # json.dump(list_of_dict_outputs, f, indent=4) + + # json.dump([obj for obj in list_of_dict_outputs], f) + + # 2024-01-05 Friday 21:11:21. THIS NOW WORKS! OUTPUT IS A LIST OF FORMATTED JSON OBJECTS! + json.dump([obj for obj in list_of_dict_outputs], f, indent=4) + + return purl_versions + + +if __name__ == "__main__": + purlcli()