Save interim drafts #247

Reference: #247

Signed-off-by: John M. Horan <[email protected]>

packagedb/tests/test_purlcli.py

@@ -1,14 +1,20 @@
+import json
 import os
 
 from click.testing import CliRunner
 from commoncode.testcase import FileBasedTesting
+from django.test import TestCase
+from fetchcode.package_versions import PackageVersion, router, versions
 
 import purlcli
+from packagedb.models import Package
 
 
 class TestPURLCLI(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), "data")
 
+    # QUESTION: These four validate tests call validate_purls(), which queries the validate endpoint.  Is that what we want here?  I think we might be able to mock the API if we pass the API to the validate_purls() function instead of defining the API inside the function as we do now.
+
     def test_validate_purl(self):
         test_purls = [
             "pkg:nginx/[email protected]?os=windows",
@@ -92,3 +98,92 @@ def test_validate_purl_strip(self):
         ]
 
         self.assertEqual(validated_purls, expected_results)
+
+    def test_versions(self):
+        purls = ["pkg:pypi/fetchcode"]
+        abc = purlcli.list_versions(purls)
+
+        print(f"\nabc = {abc}")
+
+        for purl_list in abc:
+            for p in purl_list:
+                # print(PackageVersion.to_dict(p))
+                p_dict = PackageVersion.to_dict(p)
+                # print(f"\np_dict = {p_dict}")
+                # print(f"\ntype(p_dict) = {type(p_dict)}")
+
+                json_p_dict = json.dumps(p_dict)
+                # print(f"\njson_p_dict = {json_p_dict}")
+                # print(f"\ntype(json_p_dict) = {type(json_p_dict)}")
+
+
+# 2024-01-08 Monday 17:55:15.  Based on test_api.py's class PurlValidateApiTestCase(TestCase).
+class TestPURLCLI_API(TestCase):
+    def setUp(self):
+        self.package_data = {
+            "type": "npm",
+            "namespace": "",
+            "name": "foobar",
+            "version": "1,1.0",
+            "qualifiers": "",
+            "subpath": "",
+            "download_url": "",
+            "filename": "Foo.zip",
+            "sha1": "testsha1",
+            "md5": "testmd5",
+            "size": 101,
+        }
+        self.package = Package.objects.create(**self.package_data)
+        self.package.refresh_from_db()
+
+    def test_api_purl_validation(self):
+        data1 = {
+            "purl": "pkg:npm/[email protected]",
+            "check_existence": True,
+        }
+        response1 = self.client.get(f"/api/validate/", data=data1)
+
+        print(f"\nresponse1 = {response1}")
+        print(f"\nresponse1.data = {response1.data}")
+        print(f"")
+
+        data2 = {
+            "purl": "pkg:npm/[email protected]",
+            "check_existence": True,
+        }
+        response2 = self.client.get(f"/api/validate/", data=data2)
+
+        print(f"\nresponse2 = {response2}")
+        print(f"\nresponse2.data = {response2.data}")
+        print(f"")
+
+        self.assertEquals(True, response1.data["valid"])
+        self.assertEquals(True, response1.data["exists"])
+        self.assertEquals(
+            "The provided Package URL is valid, and the package exists in the upstream repo.",
+            response1.data["message"],
+        )
+
+        self.assertEquals(False, response2.data["valid"])
+        self.assertEquals(
+            "The provided PackageURL is not valid.", response2.data["message"]
+        )
+
+        # ZZZ: 2024-01-08 Monday 18:54:51.  Some exploring:
+
+        data3 = {
+            "purl": "pkg:npm/ogdendunes",
+            "check_existence": True,
+        }
+        response3 = self.client.get(f"/api/validate/", data=data3)
+
+        print(f"\nresponse3 = {response3}")
+        print(f"\nresponse3.data = {response3.data}")
+        print(f"")
+
+        self.assertEqual(True, response3.data["valid"])
+        self.assertEqual(False, response3.data["exists"])
+        self.assertEqual(
+            "The provided Package URL is valid but does not exist in the upstream repo.",
+            response3.data["message"],
+        )

purlcli.py

@@ -2,6 +2,7 @@
 
 import click
 import requests
+from fetchcode.package_versions import PackageVersion, router, versions
 
 
 @click.group()
@@ -42,13 +43,17 @@ def validate(purls, output, file):
     if file:
         purls = file.read().splitlines(False)
 
-    validated_purls = validate_purls(purls)
+    api_query = "https://public.purldb.io/api/validate/"
+
+    # validated_purls = validate_purls(purls)
+    validated_purls = validate_purls(purls, api_query)
 
     json.dump(validated_purls, output, indent=4)
 
 
-def validate_purls(purls):
-    api_query = "https://public.purldb.io/api/validate/"
+# def validate_purls(purls):
+def validate_purls(purls, api_query):
+    # api_query = "https://public.purldb.io/api/validate/"
     validated_purls = []
     for purl in purls:
         purl = purl.strip()
@@ -62,5 +67,173 @@ def validate_purls(purls):
     return validated_purls
 
 
+@purlcli.command(name="versions")
+@click.option(
+    "--purl",
+    "purls",
+    multiple=True,
+    required=False,
+    help="PackageURL or PURL.",
+)
+@click.option(
+    "--output",
+    type=click.File(mode="w", encoding="utf-8"),
+    required=True,
+    default="-",
+    help="Write validation output as JSON to FILE.",
+)
+@click.option(
+    "--file",
+    type=click.File(mode="r", encoding="utf-8"),
+    required=False,
+    help="Read a list of PURLs from a FILE, one per line.",
+)
+def get_versions(purls, output, file):
+    """
+    Check the syntax of one or more PURLs.
+    """
+    if (purls and file) or not (purls or file):
+        raise click.UsageError("Use either purls or file but not both.")
+
+    if file:
+        purls = file.read().splitlines(False)
+
+    purl_versions = list_versions(purls)
+
+    purl_versions
+    # print(f"\nlen(purl_versions) = {len(purl_versions)}")
+
+
+def list_versions(purls):
+    print(f"\nlen(purls) = {len(purls)}")
+    print(f"\ntype(purls) = {type(purls)}")
+    print(f"\npurls = {purls}")
+    purl_versions = []
+
+    list_of_dict_outputs = []
+    # and this will hold the dict_outputs converted with json.dumps():
+    list_of_dict_output_json_dumps = []
+    for purl in purls:
+        dict_output = {}
+        print(f"\n==> purl = {purl}")
+        dict_output["purl"] = purl
+        dict_output["versions"] = []
+        purl = purl.strip()
+        if not purl:
+            continue
+
+        # This works: this is a list of PackageVersion objects
+        results = list(router.process(purl))
+
+        print(f"\n\nrouter.process(purl) = {router.process(purl)}")
+
+        print(f"\nlist(router.process(purl)) = {list(router.process(purl))}")
+
+        # versions(purl) is a generator object
+        print(f"\nversions(purl) = {versions(purl)}")
+
+        # Another test -- this is a list of PackageVersion objects
+        results_versions = list(versions(purl))
+        print(f"\nresults_versions = {results_versions}")
+
+        purl_versions.append(results)
+
+        # Test: list of strings
+        results_values = [v.value for v in router.process(purl)]
+        print(f"\nresults_values = {results_values}")
+
+        # 2024-01-05 Friday 17:25:41.  Iterate through PackageVersion() objects
+        for package_version_object in list(versions(purl)):
+            print(f"\n*** package_version_object = {package_version_object}")
+
+            print(
+                f"\n*** package_version_object.to_dict() = {package_version_object.to_dict()}"
+            )
+
+            print(
+                f"\n*** package_version_object.to_dict()['value'] = {package_version_object.to_dict()['value']}"
+            )
+
+            print(
+                f"\n*** package_version_object.to_dict()['release_date'] = {package_version_object.to_dict()['release_date']}"
+            )
+
+            # Here, too, create dict which we'll convert to JSON with json.dumps().
+            nested_dict = {}
+            print(f"type(nested_dict) = {type(nested_dict)}")
+
+            nested_purl = purl + "@" + f'{package_version_object.to_dict()["value"]}'
+            # nested_purl = "TEST"
+
+            # dict_output["versions"].append({"purl": nested_purl})
+            nested_dict["purl"] = nested_purl
+            nested_dict["version"] = f'{package_version_object.to_dict()["value"]}'
+            nested_dict[
+                "release_date"
+            ] = f'{package_version_object.to_dict()["release_date"]}'
+            dict_output["versions"].append(nested_dict)
+
+        # dict
+        print(f"\n==> dict_output = {dict_output}")
+        print(f"\n==> type(dict_output) = {type(dict_output)}")
+        # add to list
+        list_of_dict_outputs.append(dict_output)
+        # 2024-01-05 Friday 20:53:18.  Does this format?
+        # list_of_dict_outputs.append(json.dumps(dict_output, indent=2, sort_keys=False))
+
+        # 2024-01-05 Friday 18:31:54.  Let's convert dict_output with json.loads() and json.dumps() -- but that errors.  Omit json.loads()???
+        # json.dumps() works but the .json file is not formatted, i.e., no indents
+        dict_output_json_dumps = json.dumps(dict_output, indent=2, sort_keys=False)
+
+        # no the next 2 fail.
+        # dict_output_json_loads = json.loads(dict_output)
+        # dict_output_json_dumps = json.dumps(
+        #     dict_output_json_loads, indent=2, sort_keys=False
+        # )
+
+        print(f"\n==> dict_output_json_dumps = {dict_output_json_dumps}")
+        # and add to the separate list
+        list_of_dict_output_json_dumps.append(dict_output_json_dumps)
+
+    print(f"\npurl_versions = {purl_versions}")
+
+    print(f"\n==> list_of_dict_outputs = {list_of_dict_outputs}")
+    print(f"\n==> list_of_dict_output_json_dumps = {list_of_dict_output_json_dumps}")
+
+    with open(
+        "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_outputs-2024-01-08.json",
+        "w",
+    ) as f:
+        json.dump(list_of_dict_outputs, f)
+
+    with open(
+        "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-list_of_dict_output_json_dumps-2024-01-08.json",
+        "w",
+    ) as f:
+        json.dump(list_of_dict_output_json_dumps, f)
+
+    # try just one json.dumps() object -- dict_output_json_dumps -- NO: this also looks like this:
+    # "{\n  \"purl\": \"pkg:pypi/minecode\",\n  \"versions\": [\n    {\n      \"purl\":
+    with open(
+        "/mnt/c/nexb/purldb-testing/2024-01-08-testing/json-output/purlcli-single_dict_output_json_dumps-2024-01-08.json",
+        "w",
+    ) as f:
+        # json.dump(dict_output_json_dumps, f)
+        # json.dump(dict_output_json_dumps, f, indent=4)
+        # json.dump(list_of_dict_outputs, f)
+
+        # 2024-01-05 Friday 20:54:27.  This creates formatted JSON in the output file -- not in a list
+        # json.dump(dict_output, f, indent=4)
+
+        # json.dump(list_of_dict_outputs, f, indent=4)
+
+        # json.dump([obj for obj in list_of_dict_outputs], f)
+
+        # 2024-01-05 Friday 21:11:21.  THIS NOW WORKS!  OUTPUT IS A LIST OF FORMATTED JSON OBJECTS!
+        json.dump([obj for obj in list_of_dict_outputs], f, indent=4)
+
+    return purl_versions
+
+
 if __name__ == "__main__":
     purlcli()