From d7f41c3fd144def2e1c0fb4cb75bfdd1e5974f18 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 14:03:20 -0500 Subject: [PATCH 01/27] test: rename test module to be more specific and granular --- tests/{test_phylum_ci.py => test_project_metadata.py} | 2 ++ 1 file changed, 2 insertions(+) rename tests/{test_phylum_ci.py => test_project_metadata.py} (98%) diff --git a/tests/test_phylum_ci.py b/tests/test_project_metadata.py similarity index 98% rename from tests/test_phylum_ci.py rename to tests/test_project_metadata.py index 3692bbcb..ece2c49d 100644 --- a/tests/test_phylum_ci.py +++ b/tests/test_project_metadata.py @@ -1,3 +1,5 @@ +"""Test the package metadata.""" + import pathlib import sys From 4f58b706a510c7b0f9faeee514a0d927f6a0c05b Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 16:48:31 -0500 Subject: [PATCH 02/27] test: rename test module and add package metadata tests --- ...roject_metadata.py => test_package_metadata.py} | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) rename tests/{test_project_metadata.py => test_package_metadata.py} (72%) diff --git a/tests/test_project_metadata.py b/tests/test_package_metadata.py similarity index 72% rename from tests/test_project_metadata.py rename to tests/test_package_metadata.py index ece2c49d..0f2e5025 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_package_metadata.py @@ -4,7 +4,7 @@ import sys import tomli -from phylum_ci import __author__, __email__, __version__ +from phylum_ci import PKG_NAME, PKG_SUMMARY, __author__, __email__, __version__ HERE = pathlib.Path(__file__).resolve().parent PROJECT_ROOT = HERE.parent @@ -38,3 +38,15 @@ def test_author_email_metadata(): poetry_authors = PYPROJECT.get("tool", {}).get("poetry", {}).get("authors", []) assert expected_poetry_author in poetry_authors assert len(poetry_authors) == 1, "There should only be one author - the company, with it's engineering group email" + + +def test_package_name(): + """Ensure the package name is traced through from the pyproject.toml definition to the script entrypoint usage.""" + expected_pkg_name = PYPROJECT.get("tool", {}).get("poetry", {}).get("name", "") + assert expected_pkg_name == PKG_NAME + + +def test_package_description(): + """Ensure the package description is traced through from the pyproject definition to the script entrypoint usage.""" + expected_pkg_name = PYPROJECT.get("tool", {}).get("poetry", {}).get("description", "") + assert expected_pkg_name == PKG_SUMMARY From f7618bbfc66e92a8317f7f8633e068108ea9ee8f Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 16:51:22 -0500 Subject: [PATCH 03/27] refactor: update project description and make more package metadata available --- pyproject.toml | 2 +- src/phylum_ci/__init__.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b143937..b002ccd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "phylum-ci" version = "0.0.1" -description = "Utilities for Phylum integrations" +description = "Utilities for handling Phylum integrations" license = "MIT" authors = ["Phylum, Inc. "] homepage = "https://phylum.io/" diff --git a/src/phylum_ci/__init__.py b/src/phylum_ci/__init__.py index 990474e9..9c49ee94 100644 --- a/src/phylum_ci/__init__.py +++ b/src/phylum_ci/__init__.py @@ -7,7 +7,12 @@ import importlib_metadata +PKG_METADATA = importlib_metadata.metadata(__name__) + # TODO: Bump this version to at least 0.1.0 once there is more product centered functionality provided by this package __version__ = importlib_metadata.version(__name__) -__author__ = importlib_metadata.metadata(__name__).get("Author") -__email__ = importlib_metadata.metadata(__name__).get("Author-email") +__author__ = PKG_METADATA.get("Author") +__email__ = PKG_METADATA.get("Author-email") + +PKG_NAME = PKG_METADATA.get("Name") +PKG_SUMMARY = PKG_METADATA.get("Summary") From cdd33870c131bcc3c6919575b953d733167bacbb Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 16:56:11 -0500 Subject: [PATCH 04/27] feat: add ability to invoke the package entrypoint as a module With this change it is possible to call the package entrypoint as: `python -m phylum_ci` --- src/phylum_ci/__main__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/phylum_ci/__main__.py diff --git a/src/phylum_ci/__main__.py b/src/phylum_ci/__main__.py new file mode 100644 index 00000000..225ab3de --- /dev/null +++ b/src/phylum_ci/__main__.py @@ -0,0 +1,3 @@ +from phylum_ci.cli import main + +main() From 6f866bc5abdab2e5149ba8aa14dbbb4230f3da02 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 16:58:27 -0500 Subject: [PATCH 05/27] feat: add --version option to CLI --- src/phylum_ci/cli.py | 18 +++++++++++++----- tests/test_cli.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 tests/test_cli.py diff --git a/src/phylum_ci/cli.py b/src/phylum_ci/cli.py index a6f56a96..b14851e9 100644 --- a/src/phylum_ci/cli.py +++ b/src/phylum_ci/cli.py @@ -3,15 +3,23 @@ import argparse import sys +from phylum_ci import PKG_NAME, PKG_SUMMARY, __version__ -def get_args(): - """Get the arguments from the command line and return them.""" + +def get_args(args=None): + """Get the arguments from the command line and return them. + + Use `args` parameter as dependency injection for testing. + """ parser = argparse.ArgumentParser( - prog="phylum-ci", - description="CLI for handling Phylum integrations", + prog=PKG_NAME, + description=PKG_SUMMARY, formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - return parser.parse_args() + + parser.add_argument("--version", action="version", version=f"{PKG_NAME} {__version__}") + + return parser.parse_args(args) def main(): diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..3c9f64a8 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,29 @@ +""""Test the command line interface (CLI).""" + +import subprocess +import sys + +from phylum_ci import PKG_NAME, __version__ + + +def test_run_as_module(): + """Ensure the CLI can be called as a module. + + This is the `python -m ` format to "run library module as a script." + NOTE: The must be specified with an underscore, even if the corresponding + script entry point is specified with a dash. + """ + cmd_line = [sys.executable, "-m", "phylum_ci", "--help"] + ret = subprocess.run(cmd_line) + assert ret.returncode == 0, "Running the package as a module failed" + + +def test_version_option(): + """Ensure the correct program name and version is displayed when using the `--version` option.""" + # The argparse module adds a newline to the output + expected_output = f"{PKG_NAME} {__version__}\n" + cmd_line = [sys.executable, "-m", "phylum_ci", "--version"] + ret = subprocess.run(cmd_line, check=True, capture_output=True, encoding="utf-8") + assert ret.stdout == expected_output, "Output did not match expected input" + assert not ret.stderr, "Nothing should be written to stderr" + assert ret.returncode == 0, "A non-successful return code was provided" From 52dc24e9ac2de9a6655870b88f53470725d6de57 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 20:30:14 -0500 Subject: [PATCH 06/27] test: add test for script entry points and refactor test constants out --- tests/constants.py | 11 +++++++++++ tests/test_cli.py | 11 +++++++++++ tests/test_package_metadata.py | 9 +-------- 3 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 tests/constants.py diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 00000000..f50b25f8 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,11 @@ +"""Place test package constants here.""" +import pathlib + +import tomli + +HERE = pathlib.Path(__file__).resolve().parent +PROJECT_ROOT = HERE.parent +PYPROJECT_TOML_PATH = PROJECT_ROOT / "pyproject.toml" + +with open(PYPROJECT_TOML_PATH, "rb") as f: + PYPROJECT = tomli.load(f) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3c9f64a8..67db1b36 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,8 @@ from phylum_ci import PKG_NAME, __version__ +from .constants import PYPROJECT + def test_run_as_module(): """Ensure the CLI can be called as a module. @@ -18,6 +20,15 @@ def test_run_as_module(): assert ret.returncode == 0, "Running the package as a module failed" +def test_run_as_script(): + """Ensure the CLI can be called by it's script entry point.""" + scripts = PYPROJECT.get("tool", {}).get("poetry", {}).get("scripts", {}) + assert scripts, "There should be at least one script entry point" + for script in scripts: + ret = subprocess.run([script, "-h"]) + assert ret.returncode == 0, f"{script} entry point failed" + + def test_version_option(): """Ensure the correct program name and version is displayed when using the `--version` option.""" # The argparse module adds a newline to the output diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py index 0f2e5025..3b02d65f 100644 --- a/tests/test_package_metadata.py +++ b/tests/test_package_metadata.py @@ -1,17 +1,10 @@ """Test the package metadata.""" -import pathlib import sys -import tomli from phylum_ci import PKG_NAME, PKG_SUMMARY, __author__, __email__, __version__ -HERE = pathlib.Path(__file__).resolve().parent -PROJECT_ROOT = HERE.parent -PYPROJECT_TOML_PATH = PROJECT_ROOT / "pyproject.toml" - -with open(PYPROJECT_TOML_PATH, "rb") as f: - PYPROJECT = tomli.load(f) +from .constants import PYPROJECT def test_project_version(): From 413f644b6b471efa8d361a04d284aee6047114bd Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 20:42:52 -0500 Subject: [PATCH 07/27] test: clean up package metadata tests --- tests/test_package_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py index 3b02d65f..f3db013d 100644 --- a/tests/test_package_metadata.py +++ b/tests/test_package_metadata.py @@ -19,7 +19,7 @@ def test_python_version(): supported_minor_versions = (7, 8, 9, 10) python_version = sys.version_info assert python_version.major == 3, "Only Python 3 is supported" - assert python_version.minor in supported_minor_versions + assert python_version.minor in supported_minor_versions, "Attempting to run unsupported Python version" def test_author_email_metadata(): @@ -29,17 +29,17 @@ def test_author_email_metadata(): # Package authors in Poetry are specified as a list of "name " entries expected_poetry_author = f"{__author__} <{__email__}>" poetry_authors = PYPROJECT.get("tool", {}).get("poetry", {}).get("authors", []) - assert expected_poetry_author in poetry_authors + assert expected_poetry_author in poetry_authors, "Package author/email should be defined in pyproject.toml only" assert len(poetry_authors) == 1, "There should only be one author - the company, with it's engineering group email" def test_package_name(): """Ensure the package name is traced through from the pyproject.toml definition to the script entrypoint usage.""" expected_pkg_name = PYPROJECT.get("tool", {}).get("poetry", {}).get("name", "") - assert expected_pkg_name == PKG_NAME + assert expected_pkg_name == PKG_NAME, "The package name should be defined in pyproject.toml only" def test_package_description(): """Ensure the package description is traced through from the pyproject definition to the script entrypoint usage.""" - expected_pkg_name = PYPROJECT.get("tool", {}).get("poetry", {}).get("description", "") - assert expected_pkg_name == PKG_SUMMARY + expected_pkg_desc = PYPROJECT.get("tool", {}).get("poetry", {}).get("description", "") + assert expected_pkg_desc == PKG_SUMMARY, "The package description should be defined in pyproject.toml only" From 3f4215730b9d8b20765d11cf2c1bf63d1b55457a Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 18 Apr 2022 20:52:53 -0500 Subject: [PATCH 08/27] build: bump the version to 0.1.1 There is at least a little bit of functionality provided by this package now and v0.1.0 is already taken/used --- pyproject.toml | 4 ++-- src/phylum_ci/__init__.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b002ccd7..0463c2e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "phylum-ci" -version = "0.0.1" +version = "0.1.1" description = "Utilities for handling Phylum integrations" license = "MIT" authors = ["Phylum, Inc. "] @@ -16,7 +16,7 @@ keywords = ["dependency", "security", "CI", "integration"] # Classifiers can be found here: https://pypi.org/classifiers/ classifiers = [ # TODO: Update this value as the project/package matures - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", diff --git a/src/phylum_ci/__init__.py b/src/phylum_ci/__init__.py index 9c49ee94..271cbf19 100644 --- a/src/phylum_ci/__init__.py +++ b/src/phylum_ci/__init__.py @@ -9,7 +9,6 @@ PKG_METADATA = importlib_metadata.metadata(__name__) -# TODO: Bump this version to at least 0.1.0 once there is more product centered functionality provided by this package __version__ = importlib_metadata.version(__name__) __author__ = PKG_METADATA.get("Author") __email__ = PKG_METADATA.get("Author-email") From a829e99871fb63a71866dc106ea1001078ffe230 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Tue, 19 Apr 2022 15:46:48 -0500 Subject: [PATCH 09/27] refactor: rename phylum-ci package to phylum The PyPI request to take over the `phylum` package name was granted. This change makes use of that name, changing `phylum-ci` and `phylum_ci` to `phylum` everywhere that it makes sense. There are still some instances that should remain `phylum-ci` since that is what the GitHub repository is named. --- CONTRIBUTING.md | 8 ++++---- README.md | 20 ++++++++++---------- docs/release_process.md | 2 +- pyproject.toml | 6 +++--- src/{phylum_ci => phylum}/__init__.py | 2 +- src/phylum/__main__.py | 3 +++ src/{phylum_ci => phylum}/cli.py | 4 ++-- src/phylum_ci/__main__.py | 3 --- tests/test_cli.py | 6 +++--- tests/test_package_metadata.py | 2 +- 10 files changed, 28 insertions(+), 28 deletions(-) rename src/{phylum_ci => phylum}/__init__.py (93%) create mode 100644 src/phylum/__main__.py rename src/{phylum_ci => phylum}/cli.py (88%) delete mode 100644 src/phylum_ci/__main__.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1674affa..f9b2fd31 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ Look through the GitHub issues for features to work on, which will be labeled wi ### Write Documentation The `phylum-ci` project could always use more documentation, whether as part of the -official phylum-ci docs, in docstrings, or even on the web in blog posts, articles, and such. +official phylum docs, in docstrings, or even on the web in blog posts, articles, and such. ### Submit Feedback @@ -144,11 +144,11 @@ interact with `pytest` by passing additional positional arguments: ```sh # run a specific test module across all test environments -poetry run tox tests/test_phylum_ci.py +poetry run tox tests/test_package_metadata.py # run a specific test module across a specific test environment -poetry run tox -e py39 test/test_phylum_ci.py +poetry run tox -e py39 test/test_package_metadata.py # run a specific test function within a test module, in a specific test environment -poetry run tox -e py310 test/test_phylum_ci.py::test_python_version +poetry run tox -e py310 test/test_package_metadata.py::test_python_version # passing additional options to pytest requires using the double dash escape poetry run tox -e py310 -- --help ``` diff --git a/README.md b/README.md index 8d2a9ae4..ef7404cb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # phylum-ci -[![PyPI](https://img.shields.io/pypi/v/phylum-ci)](https://pypi.org/project/phylum-ci/) -![PyPI - Status](https://img.shields.io/pypi/status/phylum-ci) -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/phylum-ci)](https://pypi.org/project/phylum-ci/) +[![PyPI](https://img.shields.io/pypi/v/phylum)](https://pypi.org/project/phylum/) +![PyPI - Status](https://img.shields.io/pypi/status/phylum) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/phylum)](https://pypi.org/project/phylum/) [![GitHub](https://img.shields.io/github/license/phylum-dev/phylum-ci)](https://github.com/phylum-dev/phylum-ci/blob/main/LICENSE) [![GitHub issues](https://img.shields.io/github/issues/phylum-dev/phylum-ci)](https://github.com/phylum-dev/phylum-ci/issues) ![GitHub last commit](https://img.shields.io/github/last-commit/phylum-dev/phylum-ci) @@ -13,30 +13,30 @@ Python package for handling CI and other integrations ### Installation -The `phylum-ci` package is pip installable for the environment of your choice: +The `phylum` Python package is pip installable for the environment of your choice: ```sh -pip install phylum-ci +pip install phylum ``` It can also also be installed in an isolated environment with the excellent [`pipx` tool](https://pypa.github.io/pipx/): ```sh # Globally install the app(s) on your system in an isolated virtual environment for the package -pipx install phylum-ci +pipx install phylum # Use the apps from the package in an ephemeral environment -pipx run phylum-ci +pipx run --spec phylum phylum-install ``` It requires Python 3.7+ to run. ### Usage -The `phylum-ci` package exposes its functionality with a command line interface (CLI). To view the options available -from the CLI, print the help message: +The `phylum` Python package exposes its functionality with a command line interface (CLI). +To view the options available from the CLI, print the help message from one of the scripts provided as entry points: ```sh -phylum-ci -h +phylum-install -h ``` ## License diff --git a/docs/release_process.md b/docs/release_process.md index 38382545..bbc5aae0 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -65,7 +65,7 @@ approach, an option is exposed to optionally publish the built package to the environment. For example using `pipx` to run a specific developmental release version: ```sh -pipx run --index-url https://test.pypi.org/simple/ --spec "phylum-ci==0.0.2.dev6" phylum-ci -h +pipx run --index-url https://test.pypi.org/simple/ --spec "phylum==0.0.2.dev6" phylum-install -h ``` Currently this workflow uses the `Staging` environment, as configured in diff --git a/pyproject.toml b/pyproject.toml index 0463c2e9..fbf47526 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -name = "phylum-ci" +name = "phylum" version = "0.1.1" description = "Utilities for handling Phylum integrations" license = "MIT" @@ -31,7 +31,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", ] packages = [ - { include = "phylum_ci", from = "src" }, + { include = "phylum", from = "src" }, { include = "tests", format = "sdist" }, ] # TODO: Add include and/or exclude items here as needed @@ -42,7 +42,7 @@ packages = [ "Issue Tracker" = "https://github.com/phylum-dev/phylum-ci/issues" [tool.poetry.scripts] -phylum-ci = "phylum_ci.cli:main" +phylum-install = "phylum.cli:main" [tool.poetry.dependencies] python = "^3.7" diff --git a/src/phylum_ci/__init__.py b/src/phylum/__init__.py similarity index 93% rename from src/phylum_ci/__init__.py rename to src/phylum/__init__.py index 271cbf19..356d4de1 100644 --- a/src/phylum_ci/__init__.py +++ b/src/phylum/__init__.py @@ -1,4 +1,4 @@ -"""Top-level package for phylum-ci.""" +"""Top-level package for phylum.""" # TODO: Use only the standard library form (importlib.metadata) only after Python 3.7 support is dropped # https://github.com/phylum-dev/phylum-ci/issues/18 try: diff --git a/src/phylum/__main__.py b/src/phylum/__main__.py new file mode 100644 index 00000000..b0714838 --- /dev/null +++ b/src/phylum/__main__.py @@ -0,0 +1,3 @@ +from phylum.cli import main + +main() diff --git a/src/phylum_ci/cli.py b/src/phylum/cli.py similarity index 88% rename from src/phylum_ci/cli.py rename to src/phylum/cli.py index b14851e9..08c0ddf3 100644 --- a/src/phylum_ci/cli.py +++ b/src/phylum/cli.py @@ -1,9 +1,9 @@ -"""Console script for phylum_ci.""" +"""Console script for phylum-install.""" import argparse import sys -from phylum_ci import PKG_NAME, PKG_SUMMARY, __version__ +from phylum import PKG_NAME, PKG_SUMMARY, __version__ def get_args(args=None): diff --git a/src/phylum_ci/__main__.py b/src/phylum_ci/__main__.py deleted file mode 100644 index 225ab3de..00000000 --- a/src/phylum_ci/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from phylum_ci.cli import main - -main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 67db1b36..61e48c7c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,7 +3,7 @@ import subprocess import sys -from phylum_ci import PKG_NAME, __version__ +from phylum import PKG_NAME, __version__ from .constants import PYPROJECT @@ -15,7 +15,7 @@ def test_run_as_module(): NOTE: The must be specified with an underscore, even if the corresponding script entry point is specified with a dash. """ - cmd_line = [sys.executable, "-m", "phylum_ci", "--help"] + cmd_line = [sys.executable, "-m", "phylum", "--help"] ret = subprocess.run(cmd_line) assert ret.returncode == 0, "Running the package as a module failed" @@ -33,7 +33,7 @@ def test_version_option(): """Ensure the correct program name and version is displayed when using the `--version` option.""" # The argparse module adds a newline to the output expected_output = f"{PKG_NAME} {__version__}\n" - cmd_line = [sys.executable, "-m", "phylum_ci", "--version"] + cmd_line = [sys.executable, "-m", "phylum", "--version"] ret = subprocess.run(cmd_line, check=True, capture_output=True, encoding="utf-8") assert ret.stdout == expected_output, "Output did not match expected input" assert not ret.stderr, "Nothing should be written to stderr" diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py index f3db013d..7ca4112d 100644 --- a/tests/test_package_metadata.py +++ b/tests/test_package_metadata.py @@ -2,7 +2,7 @@ import sys -from phylum_ci import PKG_NAME, PKG_SUMMARY, __author__, __email__, __version__ +from phylum import PKG_NAME, PKG_SUMMARY, __author__, __email__, __version__ from .constants import PYPROJECT From a508c9cc71736fd53250cada671175c5ca791b67 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Tue, 19 Apr 2022 17:21:05 -0500 Subject: [PATCH 10/27] refactor: allow for multiple script entry points The plan is to have multiple script entry points. This change accounts for the package structure needed to have them and populates the first one, phylum-install. It also refactors the test package into unit and functional tests. --- pyproject.toml | 2 +- src/phylum/__main__.py | 6 +++++- src/phylum/install/__init__.py | 4 ++++ src/phylum/install/__main__.py | 3 +++ src/phylum/{ => install}/cli.py | 9 +++++---- tests/functional/__init__.py | 0 .../{test_cli.py => functional/test_install.py} | 17 +++++++++-------- tests/unit/__init__.py | 0 tests/{ => unit}/test_package_metadata.py | 6 +++--- 9 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 src/phylum/install/__init__.py create mode 100644 src/phylum/install/__main__.py rename src/phylum/{ => install}/cli.py (77%) create mode 100644 tests/functional/__init__.py rename tests/{test_cli.py => functional/test_install.py} (71%) create mode 100644 tests/unit/__init__.py rename tests/{ => unit}/test_package_metadata.py (93%) diff --git a/pyproject.toml b/pyproject.toml index fbf47526..9825453e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ packages = [ "Issue Tracker" = "https://github.com/phylum-dev/phylum-ci/issues" [tool.poetry.scripts] -phylum-install = "phylum.cli:main" +phylum-install = "phylum.install.cli:main" [tool.poetry.dependencies] python = "^3.7" diff --git a/src/phylum/__main__.py b/src/phylum/__main__.py index b0714838..5765da86 100644 --- a/src/phylum/__main__.py +++ b/src/phylum/__main__.py @@ -1,3 +1,7 @@ -from phylum.cli import main +# Default to the phylum-install entry point +from phylum.install.cli import main +# TODO: Add logic here to dynamically show the ways this package can be called as a module. +# Alternate idea: use this as a pass-through to call the true phylum CLI tool. That way, Python can be used to make +# the calls - `python -m phylum` main() diff --git a/src/phylum/install/__init__.py b/src/phylum/install/__init__.py new file mode 100644 index 00000000..cb3172b7 --- /dev/null +++ b/src/phylum/install/__init__.py @@ -0,0 +1,4 @@ +"""Package for the phylum install script.""" + +# Dynamically create the script name based on the package structure to help stay DRY +SCRIPT_NAME = __name__.replace(".", "-") diff --git a/src/phylum/install/__main__.py b/src/phylum/install/__main__.py new file mode 100644 index 00000000..63a82361 --- /dev/null +++ b/src/phylum/install/__main__.py @@ -0,0 +1,3 @@ +from phylum.install.cli import main + +main() diff --git a/src/phylum/cli.py b/src/phylum/install/cli.py similarity index 77% rename from src/phylum/cli.py rename to src/phylum/install/cli.py index 08c0ddf3..ccf7c6e0 100644 --- a/src/phylum/cli.py +++ b/src/phylum/install/cli.py @@ -3,7 +3,8 @@ import argparse import sys -from phylum import PKG_NAME, PKG_SUMMARY, __version__ +from phylum import __version__ +from phylum.install import SCRIPT_NAME def get_args(args=None): @@ -12,12 +13,12 @@ def get_args(args=None): Use `args` parameter as dependency injection for testing. """ parser = argparse.ArgumentParser( - prog=PKG_NAME, - description=PKG_SUMMARY, + prog=SCRIPT_NAME, + description="Download and install the Phylum CLI tool", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument("--version", action="version", version=f"{PKG_NAME} {__version__}") + parser.add_argument("--version", action="version", version=f"{SCRIPT_NAME} {__version__}") return parser.parse_args(args) diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_cli.py b/tests/functional/test_install.py similarity index 71% rename from tests/test_cli.py rename to tests/functional/test_install.py index 61e48c7c..9653a3b1 100644 --- a/tests/test_cli.py +++ b/tests/functional/test_install.py @@ -1,11 +1,12 @@ -""""Test the command line interface (CLI).""" +""""Test the phylum-install command line interface (CLI).""" import subprocess import sys -from phylum import PKG_NAME, __version__ +from phylum import __version__ +from phylum.install import SCRIPT_NAME -from .constants import PYPROJECT +from ..constants import PYPROJECT def test_run_as_module(): @@ -15,7 +16,7 @@ def test_run_as_module(): NOTE: The must be specified with an underscore, even if the corresponding script entry point is specified with a dash. """ - cmd_line = [sys.executable, "-m", "phylum", "--help"] + cmd_line = [sys.executable, "-m", "phylum.install", "--help"] ret = subprocess.run(cmd_line) assert ret.returncode == 0, "Running the package as a module failed" @@ -24,15 +25,15 @@ def test_run_as_script(): """Ensure the CLI can be called by it's script entry point.""" scripts = PYPROJECT.get("tool", {}).get("poetry", {}).get("scripts", {}) assert scripts, "There should be at least one script entry point" - for script in scripts: - ret = subprocess.run([script, "-h"]) - assert ret.returncode == 0, f"{script} entry point failed" + assert SCRIPT_NAME in scripts, "The phylum-install script should be a defined entry point" + ret = subprocess.run([SCRIPT_NAME, "-h"]) + assert ret.returncode == 0, f"{SCRIPT_NAME} entry point failed" def test_version_option(): """Ensure the correct program name and version is displayed when using the `--version` option.""" # The argparse module adds a newline to the output - expected_output = f"{PKG_NAME} {__version__}\n" + expected_output = f"{SCRIPT_NAME} {__version__}\n" cmd_line = [sys.executable, "-m", "phylum", "--version"] ret = subprocess.run(cmd_line, check=True, capture_output=True, encoding="utf-8") assert ret.stdout == expected_output, "Output did not match expected input" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_package_metadata.py b/tests/unit/test_package_metadata.py similarity index 93% rename from tests/test_package_metadata.py rename to tests/unit/test_package_metadata.py index 7ca4112d..9c8b7105 100644 --- a/tests/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -4,7 +4,7 @@ from phylum import PKG_NAME, PKG_SUMMARY, __author__, __email__, __version__ -from .constants import PYPROJECT +from ..constants import PYPROJECT def test_project_version(): @@ -34,12 +34,12 @@ def test_author_email_metadata(): def test_package_name(): - """Ensure the package name is traced through from the pyproject.toml definition to the script entrypoint usage.""" + """Ensure the package name is traced through from the pyproject.toml definition.""" expected_pkg_name = PYPROJECT.get("tool", {}).get("poetry", {}).get("name", "") assert expected_pkg_name == PKG_NAME, "The package name should be defined in pyproject.toml only" def test_package_description(): - """Ensure the package description is traced through from the pyproject definition to the script entrypoint usage.""" + """Ensure the package description is traced through from the pyproject definition.""" expected_pkg_desc = PYPROJECT.get("tool", {}).get("poetry", {}).get("description", "") assert expected_pkg_desc == PKG_SUMMARY, "The package description should be defined in pyproject.toml only" From 0a87a657af24423f4ffbe32d8ce8e5841c089aec Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Tue, 19 Apr 2022 20:01:02 -0500 Subject: [PATCH 11/27] build: add `requests` as a dependency --- poetry.lock | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 675e4bee..1f828fcf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -20,6 +20,25 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +[[package]] +name = "certifi" +version = "2021.10.8" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.12" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + [[package]] name = "colorama" version = "0.4.4" @@ -48,6 +67,14 @@ python-versions = ">=3.7" docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "importlib-metadata" version = "4.11.3" @@ -178,6 +205,24 @@ python-versions = "*" [package.dependencies] pytest = ">=4.0.0" +[[package]] +name = "requests" +version = "2.27.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + [[package]] name = "six" version = "1.16.0" @@ -248,6 +293,19 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "urllib3" +version = "1.26.9" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "virtualenv" version = "20.14.1" @@ -282,7 +340,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "7322723177bc093c76c9629ec772d4ffd62b67a29afd1c96b9ddb5fec6355450" +content-hash = "19b820c22d7a293722ee5a1ce8241290b022415d2efd90689d69d5beef68d587" [metadata.files] atomicwrites = [ @@ -293,6 +351,14 @@ attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] +certifi = [ + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, + {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, +] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, @@ -305,6 +371,10 @@ filelock = [ {file = "filelock-3.6.0-py3-none-any.whl", hash = "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0"}, {file = "filelock-3.6.0.tar.gz", hash = "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85"}, ] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] importlib-metadata = [ {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, @@ -345,6 +415,10 @@ pytest-github-actions-annotate-failures = [ {file = "pytest-github-actions-annotate-failures-0.1.6.tar.gz", hash = "sha256:162e2fe18b8ab24716c4c3a8d88c29aa67126dc75b4e54be54b58e6fa04653c2"}, {file = "pytest_github_actions_annotate_failures-0.1.6-py2.py3-none-any.whl", hash = "sha256:5222dfa315c49d705912826335488ac1c75946c4b06782ab9a99379a7ee3af66"}, ] +requests = [ + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -369,6 +443,10 @@ typing-extensions = [ {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] +urllib3 = [ + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, +] virtualenv = [ {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, diff --git a/pyproject.toml b/pyproject.toml index 9825453e..37c0336a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ python = "^3.7" # TODO: Remove this dependency when Python 3.7 support is removed # https://github.com/phylum-dev/phylum-ci/issues/18 importlib-metadata = {version = "^4.11.3", python = "<3.8"} +requests = "^2.27.1" [tool.poetry.dev-dependencies] pytest = "^7.1.1" From f7506e260c99ff5b0c42161f78ccea05fb21131e Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Wed, 20 Apr 2022 15:58:56 -0500 Subject: [PATCH 12/27] feat: add phylum-init script entry point and initial functionality * Add deps: cryptography, packaging, and pyyaml\n* Rename phylum.install package to phylum.init\n* Add initial functionality to fetch and install the Phylum CLI --- README.md | 4 +- docs/release_process.md | 2 +- poetry.lock | 165 +++++++++++++- pyproject.toml | 5 +- src/phylum/__main__.py | 4 +- src/phylum/{install => init}/__init__.py | 2 +- src/phylum/init/__main__.py | 3 + src/phylum/init/cli.py | 207 ++++++++++++++++++ src/phylum/install/__main__.py | 3 - src/phylum/install/cli.py | 38 ---- .../{test_install.py => test_init.py} | 9 +- 11 files changed, 386 insertions(+), 56 deletions(-) rename src/phylum/{install => init}/__init__.py (73%) create mode 100644 src/phylum/init/__main__.py create mode 100644 src/phylum/init/cli.py delete mode 100644 src/phylum/install/__main__.py delete mode 100644 src/phylum/install/cli.py rename tests/functional/{test_install.py => test_init.py} (86%) diff --git a/README.md b/README.md index ef7404cb..f086e711 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It can also also be installed in an isolated environment with the excellent [`pi # Globally install the app(s) on your system in an isolated virtual environment for the package pipx install phylum # Use the apps from the package in an ephemeral environment -pipx run --spec phylum phylum-install +pipx run --spec phylum phylum-init ``` It requires Python 3.7+ to run. @@ -36,7 +36,7 @@ The `phylum` Python package exposes its functionality with a command line interf To view the options available from the CLI, print the help message from one of the scripts provided as entry points: ```sh -phylum-install -h +phylum-init -h ``` ## License diff --git a/docs/release_process.md b/docs/release_process.md index bbc5aae0..53e344a7 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -65,7 +65,7 @@ approach, an option is exposed to optionally publish the built package to the environment. For example using `pipx` to run a specific developmental release version: ```sh -pipx run --index-url https://test.pypi.org/simple/ --spec "phylum==0.0.2.dev6" phylum-install -h +pipx run --index-url https://test.pypi.org/simple/ --spec "phylum==0.0.2.dev6" phylum-init -h ``` Currently this workflow uses the `Staging` environment, as configured in diff --git a/poetry.lock b/poetry.lock index 1f828fcf..8bc17fbf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -28,6 +28,17 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "cffi" +version = "1.15.0" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "2.0.12" @@ -47,6 +58,25 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "cryptography" +version = "36.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools_rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + [[package]] name = "distlib" version = "0.3.4" @@ -119,7 +149,7 @@ python-versions = "*" name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -161,11 +191,19 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "pyparsing" version = "3.0.8" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" +category = "main" optional = false python-versions = ">=3.6.8" @@ -205,6 +243,14 @@ python-versions = "*" [package.dependencies] pytest = ">=4.0.0" +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "requests" version = "2.27.1" @@ -340,7 +386,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "19b820c22d7a293722ee5a1ce8241290b022415d2efd90689d69d5beef68d587" +content-hash = "8ef7144684b58715d92e555d334dccdfb5b7e286d38de84af192e9b1296a3ae5" [metadata.files] atomicwrites = [ @@ -355,6 +401,58 @@ certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] +cffi = [ + {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, + {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, + {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, + {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, + {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, + {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, + {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, + {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, + {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, + {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, + {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, + {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, + {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, + {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, + {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, + {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, + {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, +] charset-normalizer = [ {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, @@ -363,6 +461,28 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +cryptography = [ + {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"}, + {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf"}, + {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86"}, + {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2"}, + {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb"}, + {file = "cryptography-36.0.2-cp36-abi3-win32.whl", hash = "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"}, + {file = "cryptography-36.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e"}, + {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51"}, + {file = "cryptography-36.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3"}, + {file = "cryptography-36.0.2.tar.gz", hash = "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9"}, +] distlib = [ {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, @@ -403,6 +523,10 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] pyparsing = [ {file = "pyparsing-3.0.8-py3-none-any.whl", hash = "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"}, {file = "pyparsing-3.0.8.tar.gz", hash = "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954"}, @@ -415,6 +539,41 @@ pytest-github-actions-annotate-failures = [ {file = "pytest-github-actions-annotate-failures-0.1.6.tar.gz", hash = "sha256:162e2fe18b8ab24716c4c3a8d88c29aa67126dc75b4e54be54b58e6fa04653c2"}, {file = "pytest_github_actions_annotate_failures-0.1.6-py2.py3-none-any.whl", hash = "sha256:5222dfa315c49d705912826335488ac1c75946c4b06782ab9a99379a7ee3af66"}, ] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, diff --git a/pyproject.toml b/pyproject.toml index 37c0336a..269a250f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ packages = [ "Issue Tracker" = "https://github.com/phylum-dev/phylum-ci/issues" [tool.poetry.scripts] -phylum-install = "phylum.install.cli:main" +phylum-init = "phylum.init.cli:main" [tool.poetry.dependencies] python = "^3.7" @@ -50,6 +50,9 @@ python = "^3.7" # https://github.com/phylum-dev/phylum-ci/issues/18 importlib-metadata = {version = "^4.11.3", python = "<3.8"} requests = "^2.27.1" +cryptography = "^36.0.2" +packaging = "^21.3" +PyYAML = "^6.0" [tool.poetry.dev-dependencies] pytest = "^7.1.1" diff --git a/src/phylum/__main__.py b/src/phylum/__main__.py index 5765da86..464187ec 100644 --- a/src/phylum/__main__.py +++ b/src/phylum/__main__.py @@ -1,5 +1,5 @@ -# Default to the phylum-install entry point -from phylum.install.cli import main +# Default to the phylum-init entry point +from phylum.init.cli import main # TODO: Add logic here to dynamically show the ways this package can be called as a module. # Alternate idea: use this as a pass-through to call the true phylum CLI tool. That way, Python can be used to make diff --git a/src/phylum/install/__init__.py b/src/phylum/init/__init__.py similarity index 73% rename from src/phylum/install/__init__.py rename to src/phylum/init/__init__.py index cb3172b7..33727a5d 100644 --- a/src/phylum/install/__init__.py +++ b/src/phylum/init/__init__.py @@ -1,4 +1,4 @@ -"""Package for the phylum install script.""" +"""Package for the phylum init script.""" # Dynamically create the script name based on the package structure to help stay DRY SCRIPT_NAME = __name__.replace(".", "-") diff --git a/src/phylum/init/__main__.py b/src/phylum/init/__main__.py new file mode 100644 index 00000000..3e779500 --- /dev/null +++ b/src/phylum/init/__main__.py @@ -0,0 +1,3 @@ +from phylum.init.cli import main + +main() diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py new file mode 100644 index 00000000..f4fa2328 --- /dev/null +++ b/src/phylum/init/cli.py @@ -0,0 +1,207 @@ +"""Console script for phylum-init.""" +import argparse +import os +import pathlib +import subprocess +import sys +import tempfile +import zipfile + +import requests +import yaml +from packaging.utils import canonicalize_version +from packaging.version import InvalidVersion, Version +from phylum import __version__ +from phylum.init import SCRIPT_NAME + +# These are the supported Rust target triples +# TODO: provide a reference +# TODO: get rid of this when the install method switches to the universal install script +TARGET_TRIPLES = ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-musl", +) +TOKEN_ENVVAR_NAME = "PHYLUM_TOKEN" +PHYLUM_PATH = pathlib.Path.home() / ".phylum" +PHYLUM_BIN_PATH = PHYLUM_PATH / "phylum" +SETTINGS_YAML_PATH = PHYLUM_PATH / "settings.yaml" + + +def version_check(version: str) -> str: + """Check a given version for validity and return a normalized form of it.""" + if version == "latest": + return version + + version = version.lower() + if not version.startswith("v"): + version = f"v{version}" + + # TODO: Check for valid versions by using the GitHub API to compare against actual releases? + # raise argparse.ArgumentTypeError(f"version {version} does not exist as a release") + + try: + # Ensure the version is at least v2.0.0, which is when the release layout structure changed + if Version("v2.0.0") > Version(canonicalize_version(version)): + raise argparse.ArgumentTypeError("version must be at least v2.0.0") + except InvalidVersion as err: + raise argparse.ArgumentTypeError("an invalid version was provided") from err + + return version + + +def save_file_from_url(url, file, mode): + """Save a file from a given URL to a local file with a given mode""" + print(f" [*] Getting {url} file ...", end="") + req = requests.get(url, timeout=2.0) + req.raise_for_status() + print("Done") + + print(f" [*] Saving {url} file to {file} ...", end="") + with open(file, mode) as f: + f.write(req.content) + print("Done") + + +def get_archive_url(version, archive_name): + """Craft an archive download URL from a given version and archive name.""" + github_base_uri = "https://github.com/phylum-dev/cli/releases" + latest_version_uri = f"{github_base_uri}/latest/download" + specific_version_uri = f"{github_base_uri}/download" + + # TODO: Use the GitHub API instead? + # GITHUB_API = "https://api.github.com" + + if version == "latest": + archive_url = f"{latest_version_uri}/{archive_name}" + else: + archive_url = f"{specific_version_uri}/{version}/{archive_name}" + + return archive_url + + +def is_token_set(token=None): + """Check if any token is already set. + + Optionally, check if a specific given `token` is set. + """ + if not SETTINGS_YAML_PATH.exists(): + return False + settings_dict = yaml.safe_load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + configured_token = settings_dict.get("auth_info", {}).get("offline_access") + if configured_token is None: + return False + if token is not None: + if token != configured_token: + return False + return True + + +def setup_token(token): + """Setup the CLI credentials with a provided token.""" + # The phylum CLI settings.yaml file won't exist upon initial install + # but running a command will trigger the CLI to generate it + if not SETTINGS_YAML_PATH.exists(): + cmd_line = [PHYLUM_BIN_PATH, "version"] + subprocess.run(cmd_line, check=True) + + settings_dict = yaml.safe_load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + settings_dict.setdefault("auth_info", {}) + settings_dict["auth_info"]["offline_access"] = token + with open(SETTINGS_YAML_PATH, "w", encoding="utf-8") as f: + yaml.dump(settings_dict, f) + + # Check that the token was setup correctly by using it to display the current auth status + cmd_line = [PHYLUM_BIN_PATH, "auth", "status"] + subprocess.run(cmd_line, check=True) + + +def get_args(args=None): + """Get the arguments from the command line and return them. + + Use `args` parameter as dependency injection for testing. + """ + parser = argparse.ArgumentParser( + prog=SCRIPT_NAME, + description="Fetch and install the Phylum CLI", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "-v", + "--phylum-version", + dest="version", + default="latest", + type=version_check, + help="the version of the Phylum CLI to install", + ) + parser.add_argument( + "-t", + "--target", + choices=TARGET_TRIPLES, + default="x86_64-unknown-linux-musl", + help="the target platform type where the CLI will be installed", + ) + parser.add_argument( + "-k", + "--phylum-token", + dest="token", + help=f"""Phylum user token. Can also specify this option's value by setting the `{TOKEN_ENVVAR_NAME}` + environment variable. The value specified with this option takes precedence when both are provided.""", + ) + # TODO: Add a --list option, to show which versions are available? + # TODO: Account for pre-releases? + # parser.add_argument("-p", "--pre-release", action="store_true", help="specify to include pre-release versions") + parser.add_argument("--version", action="version", version=f"{SCRIPT_NAME} {__version__}") + + return parser.parse_args(args) + + +def main(): + """Main entrypoint.""" + args = get_args() + + token = args.token or os.getenv(TOKEN_ENVVAR_NAME) + if not token and not is_token_set(): + raise ValueError(f"Phylum Token not supplied as option or `{TOKEN_ENVVAR_NAME}` environment variable") + + archive_name = f"phylum-{args.target}.zip" + minisig_name = f"{archive_name}.minisig" + archive_url = get_archive_url(args.version, archive_name) + minisig_url = f"{archive_url}.minisig" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir_path = pathlib.Path(temp_dir) + archive_path = temp_dir_path / archive_name + minisig_path = temp_dir_path / minisig_name + + save_file_from_url(archive_url, archive_path, "wb") + save_file_from_url(minisig_url, minisig_path, "wb") + + # TODO: Verify the download with minisign + + with zipfile.ZipFile(archive_path, mode="r") as zip_file: + if zip_file.testzip() is not None: + raise zipfile.BadZipFile(f"There was a bad file in the zip archive {archive_name}") + extracted_dir = temp_dir_path + top_level_zip_entry = zip_file.infolist()[0] + if top_level_zip_entry.is_dir(): + extracted_dir = temp_dir_path / top_level_zip_entry.filename + zip_file.extractall(path=temp_dir) + + # Run the install script + cmd_line = ["sh", "install.sh"] + subprocess.run(cmd_line, check=True, cwd=extracted_dir) + + if not is_token_set(token=token): + setup_token(token) + + # Do a check to ensure everything is working + cmd_line = [PHYLUM_BIN_PATH, "--help"] + subprocess.run(cmd_line, check=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/phylum/install/__main__.py b/src/phylum/install/__main__.py deleted file mode 100644 index 63a82361..00000000 --- a/src/phylum/install/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from phylum.install.cli import main - -main() diff --git a/src/phylum/install/cli.py b/src/phylum/install/cli.py deleted file mode 100644 index ccf7c6e0..00000000 --- a/src/phylum/install/cli.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Console script for phylum-install.""" - -import argparse -import sys - -from phylum import __version__ -from phylum.install import SCRIPT_NAME - - -def get_args(args=None): - """Get the arguments from the command line and return them. - - Use `args` parameter as dependency injection for testing. - """ - parser = argparse.ArgumentParser( - prog=SCRIPT_NAME, - description="Download and install the Phylum CLI tool", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - parser.add_argument("--version", action="version", version=f"{SCRIPT_NAME} {__version__}") - - return parser.parse_args(args) - - -def main(): - """Main entrypoint.""" - args = get_args() - if not args: - print("Returning error ...") - return 1 - - print("Returning success ...") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/functional/test_install.py b/tests/functional/test_init.py similarity index 86% rename from tests/functional/test_install.py rename to tests/functional/test_init.py index 9653a3b1..9d9c6e37 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_init.py @@ -1,10 +1,9 @@ -""""Test the phylum-install command line interface (CLI).""" - +""""Test the phylum-init command line interface (CLI).""" import subprocess import sys from phylum import __version__ -from phylum.install import SCRIPT_NAME +from phylum.init import SCRIPT_NAME from ..constants import PYPROJECT @@ -16,7 +15,7 @@ def test_run_as_module(): NOTE: The must be specified with an underscore, even if the corresponding script entry point is specified with a dash. """ - cmd_line = [sys.executable, "-m", "phylum.install", "--help"] + cmd_line = [sys.executable, "-m", "phylum.init", "--help"] ret = subprocess.run(cmd_line) assert ret.returncode == 0, "Running the package as a module failed" @@ -25,7 +24,7 @@ def test_run_as_script(): """Ensure the CLI can be called by it's script entry point.""" scripts = PYPROJECT.get("tool", {}).get("poetry", {}).get("scripts", {}) assert scripts, "There should be at least one script entry point" - assert SCRIPT_NAME in scripts, "The phylum-install script should be a defined entry point" + assert SCRIPT_NAME in scripts, f"The {SCRIPT_NAME} script should be a defined entry point" ret = subprocess.run([SCRIPT_NAME, "-h"]) assert ret.returncode == 0, f"{SCRIPT_NAME} entry point failed" From e50c5438099c09e38fe9027df1fbff70d47cfa33 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Wed, 20 Apr 2022 17:41:51 -0500 Subject: [PATCH 13/27] build: swap pyyaml dependency for ruamel.yaml The pyyaml dependency fails to install in Python 3.7 environments for Apple Silicon systems. This appears to be due to the default nature in which the sdist is attempted to be installed by Poetry and with the LibYAML c bindings. There is no apparent method to direct Poetry to use the `--without-libyaml` option when building from source. One advantage of switching to ruamel.yaml is that round-tripping is supported, which means order and comments are preserved with loading-modifying-dumping a YAML file. --- poetry.lock | 99 +++++++++++++++++++++++------------------- pyproject.toml | 3 +- src/phylum/init/cli.py | 8 ++-- 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8bc17fbf..04c1c6a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -243,14 +243,6 @@ python-versions = "*" [package.dependencies] pytest = ">=4.0.0" -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - [[package]] name = "requests" version = "2.27.1" @@ -269,6 +261,29 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +[[package]] +name = "ruamel.yaml" +version = "0.17.21" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.6", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel.yaml.clib" +version = "0.2.6" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "six" version = "1.16.0" @@ -386,7 +401,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "8ef7144684b58715d92e555d334dccdfb5b7e286d38de84af192e9b1296a3ae5" +content-hash = "249e791c36e3cb4a440c693206985487f860e46c9dc75061460044216312bc7d" [metadata.files] atomicwrites = [ @@ -539,45 +554,41 @@ pytest-github-actions-annotate-failures = [ {file = "pytest-github-actions-annotate-failures-0.1.6.tar.gz", hash = "sha256:162e2fe18b8ab24716c4c3a8d88c29aa67126dc75b4e54be54b58e6fa04653c2"}, {file = "pytest_github_actions_annotate_failures-0.1.6-py2.py3-none-any.whl", hash = "sha256:5222dfa315c49d705912826335488ac1c75946c4b06782ab9a99379a7ee3af66"}, ] -pyyaml = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] +"ruamel.yaml" = [ + {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, + {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, +] +"ruamel.yaml.clib" = [ + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win32.whl", hash = "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee"}, + {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"}, + {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index 269a250f..9cf53eef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ packages = [ [tool.poetry.urls] "Issue Tracker" = "https://github.com/phylum-dev/phylum-ci/issues" +"CI" = "https://github.com/phylum-dev/phylum-ci/actions" [tool.poetry.scripts] phylum-init = "phylum.init.cli:main" @@ -52,7 +53,7 @@ importlib-metadata = {version = "^4.11.3", python = "<3.8"} requests = "^2.27.1" cryptography = "^36.0.2" packaging = "^21.3" -PyYAML = "^6.0" +"ruamel.yaml" = "^0.17.21" [tool.poetry.dev-dependencies] pytest = "^7.1.1" diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index f4fa2328..e892487c 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -8,11 +8,11 @@ import zipfile import requests -import yaml from packaging.utils import canonicalize_version from packaging.version import InvalidVersion, Version from phylum import __version__ from phylum.init import SCRIPT_NAME +from ruamel.yaml import YAML # These are the supported Rust target triples # TODO: provide a reference @@ -87,7 +87,8 @@ def is_token_set(token=None): """ if not SETTINGS_YAML_PATH.exists(): return False - settings_dict = yaml.safe_load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + yaml = YAML() + settings_dict = yaml.load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) configured_token = settings_dict.get("auth_info", {}).get("offline_access") if configured_token is None: return False @@ -105,7 +106,8 @@ def setup_token(token): cmd_line = [PHYLUM_BIN_PATH, "version"] subprocess.run(cmd_line, check=True) - settings_dict = yaml.safe_load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + yaml = YAML() + settings_dict = yaml.load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) settings_dict.setdefault("auth_info", {}) settings_dict["auth_info"]["offline_access"] = token with open(SETTINGS_YAML_PATH, "w", encoding="utf-8") as f: From ac97b03f40ff140cbb612dc65dbc90e94cc05628 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Wed, 20 Apr 2022 22:47:11 -0500 Subject: [PATCH 14/27] refactor: detect the target triple automatically --- src/phylum/init/cli.py | 55 +++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index e892487c..fb2f7139 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -2,6 +2,7 @@ import argparse import os import pathlib +import platform import subprocess import sys import tempfile @@ -17,7 +18,7 @@ # These are the supported Rust target triples # TODO: provide a reference # TODO: get rid of this when the install method switches to the universal install script -TARGET_TRIPLES = ( +SUPPORTED_TARGET_TRIPLES = ( "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-musl", @@ -28,7 +29,7 @@ SETTINGS_YAML_PATH = PHYLUM_PATH / "settings.yaml" -def version_check(version: str) -> str: +def version_check(version): """Check a given version for validity and return a normalized form of it.""" if version == "latest": return version @@ -50,15 +51,43 @@ def version_check(version: str) -> str: return version -def save_file_from_url(url, file, mode): - """Save a file from a given URL to a local file with a given mode""" +def get_target_triple(): + """Get the "target triple" from the current system and return it. + + Targets are identified by their "target triple" which is the string to inform the compiler what kind of output + should be produced. A target triple consists of three strings separated by a hyphen, with a possible fourth string + at the end preceded by a hyphen. The first is the architecture, the second is the "vendor", the third is the OS + type, and the optional fourth is environment type. + + References: + * https://doc.rust-lang.org/nightly/rustc/platform-support.html + * https://rust-lang.github.io/rfcs/0131-target-specification.html + """ + # Keys are lowercase machine hardware names as returned from `uname -m`. Values are the mapped rustc architecture. + supported_arches = { + "arm64": "aarch64XXX", + "amd64": "x86_64", + } + # Keys are lowercase operating system name as returned from `uname -s`. + # Values are the mapped rustc platform, which is the vendor-os_type[-environment_type] + supported_platforms = { + "linux": "unknown-linux-musl", + "darwin": "apple-darwin", + } + arch = supported_arches.get(platform.uname().machine.lower(), "unknown") + plat = supported_platforms.get(platform.uname().system.lower(), "unknown") + return f"{arch}-{plat}" + + +def save_file_from_url(url, path): + """Save a file from a given URL to a local file path, in binary mode.""" print(f" [*] Getting {url} file ...", end="") req = requests.get(url, timeout=2.0) req.raise_for_status() print("Done") - print(f" [*] Saving {url} file to {file} ...", end="") - with open(file, mode) as f: + print(f" [*] Saving {url} file to {path} ...", end="") + with open(path, "wb") as f: f.write(req.content) print("Done") @@ -140,8 +169,8 @@ def get_args(args=None): parser.add_argument( "-t", "--target", - choices=TARGET_TRIPLES, - default="x86_64-unknown-linux-musl", + choices=SUPPORTED_TARGET_TRIPLES, + default=get_target_triple(), help="the target platform type where the CLI will be installed", ) parser.add_argument( @@ -167,7 +196,11 @@ def main(): if not token and not is_token_set(): raise ValueError(f"Phylum Token not supplied as option or `{TOKEN_ENVVAR_NAME}` environment variable") - archive_name = f"phylum-{args.target}.zip" + target_triple = args.target + if target_triple not in SUPPORTED_TARGET_TRIPLES: + raise ValueError(f"The identified target triple `{target_triple}` is not currently supported") + + archive_name = f"phylum-{target_triple}.zip" minisig_name = f"{archive_name}.minisig" archive_url = get_archive_url(args.version, archive_name) minisig_url = f"{archive_url}.minisig" @@ -177,8 +210,8 @@ def main(): archive_path = temp_dir_path / archive_name minisig_path = temp_dir_path / minisig_name - save_file_from_url(archive_url, archive_path, "wb") - save_file_from_url(minisig_url, minisig_path, "wb") + save_file_from_url(archive_url, archive_path) + save_file_from_url(minisig_url, minisig_path) # TODO: Verify the download with minisign From ee527cb54cdca6b089740561834e7ab6d005b3c8 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 00:29:35 -0500 Subject: [PATCH 15/27] refactor: pull constants from `get_target_triple` function up and reformat --- src/phylum/init/cli.py | 59 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index fb2f7139..a97ad358 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -15,19 +15,41 @@ from phylum.init import SCRIPT_NAME from ruamel.yaml import YAML -# These are the supported Rust target triples -# TODO: provide a reference -# TODO: get rid of this when the install method switches to the universal install script +# These are the currently supported Rust target triples +# +# Targets are identified by their "target triple" which is the string to inform the compiler what kind of output +# should be produced. A target triple consists of three strings separated by a hyphen, with a possible fourth string +# at the end preceded by a hyphen. The first is the architecture, the second is the "vendor", the third is the OS +# type, and the optional fourth is environment type. +# +# References: +# * https://doc.rust-lang.org/nightly/rustc/platform-support.html +# * https://rust-lang.github.io/rfcs/0131-target-specification.html SUPPORTED_TARGET_TRIPLES = ( "aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-musl", ) +# Keys are lowercase machine hardware names as returned from `uname -m`. +# Values are the mapped rustc architecture. +SUPPORTED_ARCHES = { + "arm64": "aarch64", + "amd64": "x86_64", +} +# Keys are lowercase operating system name as returned from `uname -s`. +# Values are the mapped rustc platform, which is the vendor-os_type[-environment_type]. +SUPPORTED_PLATFORMS = { + "linux": "unknown-linux-musl", + "darwin": "apple-darwin", +} + TOKEN_ENVVAR_NAME = "PHYLUM_TOKEN" PHYLUM_PATH = pathlib.Path.home() / ".phylum" PHYLUM_BIN_PATH = PHYLUM_PATH / "phylum" SETTINGS_YAML_PATH = PHYLUM_PATH / "settings.yaml" +# TODO: Add logging support, a verbosity option to control it, and swap out print statements for logging + def version_check(version): """Check a given version for validity and return a normalized form of it.""" @@ -52,30 +74,9 @@ def version_check(version): def get_target_triple(): - """Get the "target triple" from the current system and return it. - - Targets are identified by their "target triple" which is the string to inform the compiler what kind of output - should be produced. A target triple consists of three strings separated by a hyphen, with a possible fourth string - at the end preceded by a hyphen. The first is the architecture, the second is the "vendor", the third is the OS - type, and the optional fourth is environment type. - - References: - * https://doc.rust-lang.org/nightly/rustc/platform-support.html - * https://rust-lang.github.io/rfcs/0131-target-specification.html - """ - # Keys are lowercase machine hardware names as returned from `uname -m`. Values are the mapped rustc architecture. - supported_arches = { - "arm64": "aarch64XXX", - "amd64": "x86_64", - } - # Keys are lowercase operating system name as returned from `uname -s`. - # Values are the mapped rustc platform, which is the vendor-os_type[-environment_type] - supported_platforms = { - "linux": "unknown-linux-musl", - "darwin": "apple-darwin", - } - arch = supported_arches.get(platform.uname().machine.lower(), "unknown") - plat = supported_platforms.get(platform.uname().system.lower(), "unknown") + """Get the "target triple" from the current system and return it.""" + arch = SUPPORTED_ARCHES.get(platform.uname().machine.lower(), "unknown") + plat = SUPPORTED_PLATFORMS.get(platform.uname().system.lower(), "unknown") return f"{arch}-{plat}" @@ -147,7 +148,7 @@ def setup_token(token): subprocess.run(cmd_line, check=True) -def get_args(args=None): +def get_args(): """Get the arguments from the command line and return them. Use `args` parameter as dependency injection for testing. @@ -185,7 +186,7 @@ def get_args(args=None): # parser.add_argument("-p", "--pre-release", action="store_true", help="specify to include pre-release versions") parser.add_argument("--version", action="version", version=f"{SCRIPT_NAME} {__version__}") - return parser.parse_args(args) + return parser.parse_args() def main(): From 30acd30019b534e1a2bca23d01ca38165a07f9e5 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 00:30:54 -0500 Subject: [PATCH 16/27] feat: verify file downloads with their `.minisig` signature files --- src/phylum/init/cli.py | 3 +- src/phylum/init/sig.py | 108 +++++++++++++++++++++++++++++++++++++++++ tests/unit/test_sig.py | 28 +++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 src/phylum/init/sig.py create mode 100644 tests/unit/test_sig.py diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index a97ad358..dca4833d 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -13,6 +13,7 @@ from packaging.version import InvalidVersion, Version from phylum import __version__ from phylum.init import SCRIPT_NAME +from phylum.init.sig import verify_minisig from ruamel.yaml import YAML # These are the currently supported Rust target triples @@ -214,7 +215,7 @@ def main(): save_file_from_url(archive_url, archive_path) save_file_from_url(minisig_url, minisig_path) - # TODO: Verify the download with minisign + verify_minisig(archive_path, minisig_path) with zipfile.ZipFile(archive_path, mode="r") as zip_file: if zip_file.testzip() is not None: diff --git a/src/phylum/init/sig.py b/src/phylum/init/sig.py new file mode 100644 index 00000000..a7a7c0c1 --- /dev/null +++ b/src/phylum/init/sig.py @@ -0,0 +1,108 @@ +"""Helper functions for verifying minisign signatures. + +This module is meant to be a quick and dirty means of verifying minisign signatures in Python. +There is no readily accessible Python library at this time. The `py-minisign` repository exists +on GitHub to attempt this - https://github.com/x13a/py-minisign - but it does not exist as a +package on PyPI and also does not appear to be actively maintained. There is a `minisign` package +on PyPI - https://pypi.org/project/minisign/ - but it comes from a different repo and has no +functionality at the time of this writing. Short of forking the `py-minisign` repo to maintain it +and publish a package from it on PyPI, the actual format for signatures and public keys is simple +and so is verifying signatures. + +Minisign reference: https://jedisct1.github.io/minisign/ +""" +import base64 + +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.hazmat.primitives.asymmetric import ed25519 + +# This is the Minisign Public Key for Phylum, Inc. The matching private key was used to sign the software releases +PHYLUM_MINISIGN_PUBKEY = "RWT6G44ykbS8GABiLXrJrYsap7FCY77m/Jyi0fgsr/Fsy3oLwU4l0IDf" + +# The format for a minisign public key is: +# +# base64( || || ) +# +# signature_algorithm: `Ed` +# key_id: 8 random bytes +# public_key: Ed25519 public key +PHYLUM_MINISIGN_PUBKEY_SIG_ALGO = base64.b64decode(PHYLUM_MINISIGN_PUBKEY)[:2] +PHYLUM_MINISIGN_PUBKEY_KEY_ID = base64.b64decode(PHYLUM_MINISIGN_PUBKEY)[2:10] +PHYLUM_MINISIGN_PUBKEY_ED25519 = base64.b64decode(PHYLUM_MINISIGN_PUBKEY)[10:] + + +def verify_minisig(file_path, sig_path): + """Verify a given file has a valid minisign signature. + + `file_path` is the path to the file data to verify. + `sig_path` is the path to the `.minisig` file containing the minisign signature information. + + The public key is an assumed constant, the Minisign Public Key for Phylum, Inc. + """ + try: + phylum_public_key = ed25519.Ed25519PublicKey.from_public_bytes(PHYLUM_MINISIGN_PUBKEY_ED25519) + except UnsupportedAlgorithm as err: + raise RuntimeError("Ed25519 algorithm is not supported by the OpenSSL version `cryptography` is using") from err + + signature_algorithm, key_id, signature, trusted_comment, global_signature = extract_minisig_elements(sig_path) + + if signature_algorithm != b"Ed": + raise RuntimeError("Only the legacy `Ed` signature algorithm is used by Phylum currently") + + if key_id != PHYLUM_MINISIGN_PUBKEY_KEY_ID: + raise RuntimeError("The `key_id` from the `.minisig` signature did not match the `key_id` from the public key") + + # Confirm the trusted comment in the sig_path with the `global_signature` there + try: + phylum_public_key.verify(global_signature, signature + trusted_comment) + except InvalidSignature as err: + raise RuntimeError("The signature could not be verified") from err + + # Confirm the data from file_path with the signature from the .minisig `sig_path` + with open(file_path, "rb") as f: + file_data = f.read() + try: + phylum_public_key.verify(signature, file_data) + except InvalidSignature as err: + raise RuntimeError("The signature could not be verified") from err + + +def extract_minisig_elements(sig_path): + """Extract the elements from a given minisig signature file and return them.""" + # The format for a minisign signature is: + # + # untrusted comment: + # base64( || || ) + # trusted_comment: + # base64() + # + # where each line above represents a line from the `.minisig` file and the elements are defined as: + # + # signature_algorithm: `Ed` (legacy) or `ED` (hashed) + # key_id: 8 random bytes, matching the public key + # signature (legacy): ed25519() + # signature (prehashed): ed25519(Blake2b-512()) + # global_signature: ed25519( || ) + trusted_comment_prefix = "trusted comment: " + trusted_comment_prefix_len = len(trusted_comment_prefix) + ed25519_signature_len = 64 + + with open(sig_path, "rb") as f: + lines = f.read().splitlines() + if len(lines) not in (4, 5): + raise RuntimeError("The .minisig file format expects 4 lines, with an optional blank 5th line") + + decoded_sig_line = base64.b64decode(lines[1]) + signature_algorithm = decoded_sig_line[:2] + key_id = decoded_sig_line[2:10] + signature = decoded_sig_line[10:] + if len(signature) != ed25519_signature_len: + raise RuntimeError(f"The decoded signature was not {ed25519_signature_len} bytes long") + + trusted_comment = lines[2][trusted_comment_prefix_len:] + + global_signature = base64.b64decode(lines[3]) + if len(global_signature) != ed25519_signature_len: + raise RuntimeError(f"The global signature was not {ed25519_signature_len} bytes long") + + return signature_algorithm, key_id, signature, trusted_comment, global_signature diff --git a/tests/unit/test_sig.py b/tests/unit/test_sig.py new file mode 100644 index 00000000..920abf8e --- /dev/null +++ b/tests/unit/test_sig.py @@ -0,0 +1,28 @@ +"""Test the minisign signature verification module.""" +from phylum.init import sig + + +def test_phylum_minisign_pubkey(): + """Ensure the minisign public key in use by Phylum has not changed.""" + expected_key = "RWT6G44ykbS8GABiLXrJrYsap7FCY77m/Jyi0fgsr/Fsy3oLwU4l0IDf" + assert sig.PHYLUM_MINISIGN_PUBKEY == expected_key, "The key should not be changing" + + +def test_phylum_pubkey_sig_algo(): + """Ensure the Phylum minisign public key signature algorithm is `Ed` (legacy).""" + assert isinstance(sig.PHYLUM_MINISIGN_PUBKEY_SIG_ALGO, bytes) + assert sig.PHYLUM_MINISIGN_PUBKEY_SIG_ALGO == b"Ed", "Only the legacy `Ed` signature is used by Phylum currently" + + +def test_phylum_pubkey_key_id(): + """Ensure the Phylum minisign public key `key_id` has not changed.""" + expected_key_id = b"\xfa\x1b\x8e2\x91\xb4\xbc\x18" + assert isinstance(sig.PHYLUM_MINISIGN_PUBKEY_KEY_ID, bytes) + assert sig.PHYLUM_MINISIGN_PUBKEY_KEY_ID == expected_key_id, "The key ID should not be changing" + + +def test_phylum_ed25519_pubkey(): + """Ensure the Phylum minisign Ed25519 public key has not changed.""" + expected_key = b"\x00b-z\xc9\xad\x8b\x1a\xa7\xb1Bc\xbe\xe6\xfc\x9c\xa2\xd1\xf8,\xaf\xf1l\xcbz\x0b\xc1N%\xd0\x80\xdf" + assert isinstance(sig.PHYLUM_MINISIGN_PUBKEY_ED25519, bytes) + assert sig.PHYLUM_MINISIGN_PUBKEY_ED25519 == expected_key, "The Ed25519 public key should not be changing" From 709c763720ec1f1a6b5bc68f8448d529e16595df Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 11:25:52 -0500 Subject: [PATCH 17/27] style: add signature verification assumptions and format throughout --- src/phylum/init/cli.py | 41 ++++++++++++++++++----------------- src/phylum/init/sig.py | 16 +++++++++++--- tests/functional/test_init.py | 3 +-- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index dca4833d..571b3281 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -49,7 +49,13 @@ PHYLUM_BIN_PATH = PHYLUM_PATH / "phylum" SETTINGS_YAML_PATH = PHYLUM_PATH / "settings.yaml" -# TODO: Add logging support, a verbosity option to control it, and swap out print statements for logging +# Potential features to add: +# * Add logging support, a verbosity option to control it, and swap out print statements for logging +# * Check for valid versions by using the GitHub API to compare against actual releases +# * If so, also programmatically generate the SUPPORTED_TARGET_TRIPLES +# * Use the GitHub API ("https://api.github.com") to more programmatically get the archive URL +# * Add a `--list` option, to show which versions are available +# * Add an option to account for pre-releases def version_check(version): @@ -61,11 +67,8 @@ def version_check(version): if not version.startswith("v"): version = f"v{version}" - # TODO: Check for valid versions by using the GitHub API to compare against actual releases? - # raise argparse.ArgumentTypeError(f"version {version} does not exist as a release") - try: - # Ensure the version is at least v2.0.0, which is when the release layout structure changed + # The release layout structure changed starting with v2.0.0 and support here is only for the new layout if Version("v2.0.0") > Version(canonicalize_version(version)): raise argparse.ArgumentTypeError("version must be at least v2.0.0") except InvalidVersion as err: @@ -100,9 +103,6 @@ def get_archive_url(version, archive_name): latest_version_uri = f"{github_base_uri}/latest/download" specific_version_uri = f"{github_base_uri}/download" - # TODO: Use the GitHub API instead? - # GITHUB_API = "https://api.github.com" - if version == "latest": archive_url = f"{latest_version_uri}/{archive_name}" else: @@ -112,20 +112,23 @@ def get_archive_url(version, archive_name): def is_token_set(token=None): - """Check if any token is already set. + """Check if any token is already set in the CLI configuration file. Optionally, check if a specific given `token` is set. """ if not SETTINGS_YAML_PATH.exists(): return False + yaml = YAML() settings_dict = yaml.load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) configured_token = settings_dict.get("auth_info", {}).get("offline_access") + if configured_token is None: return False if token is not None: if token != configured_token: return False + return True @@ -150,10 +153,7 @@ def setup_token(token): def get_args(): - """Get the arguments from the command line and return them. - - Use `args` parameter as dependency injection for testing. - """ + """Get the arguments from the command line and return them.""" parser = argparse.ArgumentParser( prog=SCRIPT_NAME, description="Fetch and install the Phylum CLI", @@ -180,12 +180,14 @@ def get_args(): "--phylum-token", dest="token", help=f"""Phylum user token. Can also specify this option's value by setting the `{TOKEN_ENVVAR_NAME}` - environment variable. The value specified with this option takes precedence when both are provided.""", + environment variable. The value specified with this option takes precedence when both are provided. + Leave this option unspecified to use an existing token already set in the Phylum config file.""", + ) + parser.add_argument( + "--version", + action="version", + version=f"{SCRIPT_NAME} {__version__}", ) - # TODO: Add a --list option, to show which versions are available? - # TODO: Account for pre-releases? - # parser.add_argument("-p", "--pre-release", action="store_true", help="specify to include pre-release versions") - parser.add_argument("--version", action="version", version=f"{SCRIPT_NAME} {__version__}") return parser.parse_args() @@ -226,14 +228,13 @@ def main(): extracted_dir = temp_dir_path / top_level_zip_entry.filename zip_file.extractall(path=temp_dir) - # Run the install script cmd_line = ["sh", "install.sh"] subprocess.run(cmd_line, check=True, cwd=extracted_dir) if not is_token_set(token=token): setup_token(token) - # Do a check to ensure everything is working + # Check to ensure everything is working cmd_line = [PHYLUM_BIN_PATH, "--help"] subprocess.run(cmd_line, check=True) diff --git a/src/phylum/init/sig.py b/src/phylum/init/sig.py index a7a7c0c1..6ea7c8e7 100644 --- a/src/phylum/init/sig.py +++ b/src/phylum/init/sig.py @@ -5,11 +5,21 @@ on GitHub to attempt this - https://github.com/x13a/py-minisign - but it does not exist as a package on PyPI and also does not appear to be actively maintained. There is a `minisign` package on PyPI - https://pypi.org/project/minisign/ - but it comes from a different repo and has no -functionality at the time of this writing. Short of forking the `py-minisign` repo to maintain it -and publish a package from it on PyPI, the actual format for signatures and public keys is simple -and so is verifying signatures. +functionality at the time of this writing. + +Short of forking the `py-minisign` repo to maintain it and publish a package from it on PyPI, +the actual format for signatures and public keys is simple and so is verifying signatures. Minisign reference: https://jedisct1.github.io/minisign/ + +Even still, this module is NOT meant to be used as a library or general purpose minisign +signature verification. It is purpose written to specifically verify minisign signatures that +were created by Phylum. As such, it makes a number of assumptions: + +* The Minisign Public Key for Phylum, Inc. will not change between releases +* The files to be verified were created by Phylum, Inc. +* The `.minisig` signature includes a trusted comment and will therefore contain a known number of lines +* The source of the `.minisig` signature is a trusted location, controlled by Phylum, Inc. for it's CLI releases """ import base64 diff --git a/tests/functional/test_init.py b/tests/functional/test_init.py index 9d9c6e37..45fa2cf1 100644 --- a/tests/functional/test_init.py +++ b/tests/functional/test_init.py @@ -12,8 +12,7 @@ def test_run_as_module(): """Ensure the CLI can be called as a module. This is the `python -m ` format to "run library module as a script." - NOTE: The must be specified with an underscore, even if the corresponding - script entry point is specified with a dash. + NOTE: The is specified as the dotted path to the package where the `__main__.py` module exists. """ cmd_line = [sys.executable, "-m", "phylum.init", "--help"] ret = subprocess.run(cmd_line) From d40b521372aff13e949d7ff3d33c89aac93883e6 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 13:50:45 -0500 Subject: [PATCH 18/27] refactor: allow for existing tokens and encapsulate the handling of the token option Making this change means the script won't fail if the `--token` option is not specified. This means the script can be used to install or re-install on systems that already have a registered user and existing settings. --- src/phylum/init/cli.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index 571b3281..ceeb1a41 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -132,6 +132,34 @@ def is_token_set(token=None): return True +def process_token_option(token_option): + """Process the token option as parsed from the arguments.""" + # The option takes precedence over the matching environment variable. + token = os.getenv(TOKEN_ENVVAR_NAME) + if token_option is not None: + token = token_option + + if token: + print(f" [+] Phylum token supplied as an option or `{TOKEN_ENVVAR_NAME}` environment variable") + if is_token_set(): + print(" [+] An existing token is already set") + if is_token_set(token=token): + print(" [+] Supplied token matches existing token") + else: + print(" [!] Supplied token will be used to overwrite the existing token") + else: + print(" [+] No existing token exists. Supplied token will be used.") + else: + print(f" [+] Phylum token NOT supplied as option or `{TOKEN_ENVVAR_NAME}` environment variable") + if is_token_set(): + print(" [+] Existing token found. It will be used without modification.") + else: + print(" [!] Existing token not found. Use `phylum auth login` or `phylum auth register` command to set it.") + + if token and not is_token_set(token=token): + setup_token(token) + + def setup_token(token): """Setup the CLI credentials with a provided token.""" # The phylum CLI settings.yaml file won't exist upon initial install @@ -181,7 +209,9 @@ def get_args(): dest="token", help=f"""Phylum user token. Can also specify this option's value by setting the `{TOKEN_ENVVAR_NAME}` environment variable. The value specified with this option takes precedence when both are provided. - Leave this option unspecified to use an existing token already set in the Phylum config file.""", + Leave this option and it's related environment variable unspecified to either (1) use an existing token + already set in the Phylum config file or (2) to manually populate the token with a `phylum auth login` or + `phylum auth register` command after install.""", ) parser.add_argument( "--version", @@ -196,10 +226,6 @@ def main(): """Main entrypoint.""" args = get_args() - token = args.token or os.getenv(TOKEN_ENVVAR_NAME) - if not token and not is_token_set(): - raise ValueError(f"Phylum Token not supplied as option or `{TOKEN_ENVVAR_NAME}` environment variable") - target_triple = args.target if target_triple not in SUPPORTED_TARGET_TRIPLES: raise ValueError(f"The identified target triple `{target_triple}` is not currently supported") @@ -231,8 +257,7 @@ def main(): cmd_line = ["sh", "install.sh"] subprocess.run(cmd_line, check=True, cwd=extracted_dir) - if not is_token_set(token=token): - setup_token(token) + process_token_option(args.token) # Check to ensure everything is working cmd_line = [PHYLUM_BIN_PATH, "--help"] From fdd24df4f634c00057295a0a5bc49e7ac13f3a40 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 13:51:34 -0500 Subject: [PATCH 19/27] docs: add usage information for the `phylum-init` script --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f086e711..b8ed7e2c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pipx run --spec phylum phylum-init It requires Python 3.7+ to run. ### Usage - + The `phylum` Python package exposes its functionality with a command line interface (CLI). To view the options available from the CLI, print the help message from one of the scripts provided as entry points: @@ -39,6 +39,24 @@ To view the options available from the CLI, print the help message from one of t phylum-init -h ``` +The functionality can also be accessed by calling the module: + +```sh +python -m phylum.init -h +# The top level package is redirected to the phylum.init package: +python -m phylum -h +``` + +#### `phylum-init` + +The `phylum-init` script can be used to fetch and install the Phylum CLI. +It will attempt to install the latest released version of the CLI but can be specified to fetch a specific version. +It will attempt to automatically determine the correct CLI release, based on the platform where the script is run, but +a specific release target can be specified. +It will accept a Phylum token from an environment variable or specified as an option, but will also function in the case +that no token is provided. This can be because there is already a token set that should continue to be used or because +no token exists and one will need to be manually created or set, after the CLI is installed. + ## License MIT - with complete text available in the [LICENSE](LICENSE) file. From 8c4c3cbad56160214819975e3ea57c220da2bbd0 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 14:06:59 -0500 Subject: [PATCH 20/27] docs: add tip about using poetry to run script entry points --- CONTRIBUTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9b2fd31..88cac934 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -152,3 +152,12 @@ poetry run tox -e py310 test/test_package_metadata.py::test_python_version # passing additional options to pytest requires using the double dash escape poetry run tox -e py310 -- --help ``` + +To run a script entry point with the local checkout of the code (in develop mode), use `poetry`: + +```sh +# If not done previously, ensure the project is installed by poetry (only required once) +poetry install +# Use the `poetry run` command to ensure the installed project is used +poetry run phylum-init -h +``` From 15b2abd9fb4c02c7ea6b7610c988c6f7696803be Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 14:07:31 -0500 Subject: [PATCH 21/27] build: update to latest current dependencies --- poetry.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index 04c1c6a9..2b9deb46 100644 --- a/poetry.lock +++ b/poetry.lock @@ -124,7 +124,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [[package]] name = "importlib-resources" -version = "5.7.0" +version = "5.7.1" description = "Read resources from Python packages" category = "dev" optional = false @@ -158,15 +158,15 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "platformdirs" -version = "2.5.1" +version = "2.5.2" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] [[package]] name = "pluggy" @@ -348,11 +348,11 @@ testing = ["coverage (<6)", "flake8 (>=3,<4)", "pytest-cov (>=2,<3)", "pytest-mo [[package]] name = "typing-extensions" -version = "4.1.1" -description = "Backported and Experimental Type Hints for Python 3.6+" +version = "4.2.0" +description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [[package]] name = "urllib3" @@ -515,8 +515,8 @@ importlib-metadata = [ {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, ] importlib-resources = [ - {file = "importlib_resources-5.7.0-py3-none-any.whl", hash = "sha256:9c4c12f9ef4329a00c1f72f30bddb4f10e582766b8705980bb76356b3ba8bc91"}, - {file = "importlib_resources-5.7.0.tar.gz", hash = "sha256:f6a4a9949f36ae289facec8dac1a899a54cbaf6a135cc8552d2c8b69209c06a3"}, + {file = "importlib_resources-5.7.1-py3-none-any.whl", hash = "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"}, + {file = "importlib_resources-5.7.1.tar.gz", hash = "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -527,8 +527,8 @@ packaging = [ {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] platformdirs = [ - {file = "platformdirs-2.5.1-py3-none-any.whl", hash = "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"}, - {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, + {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, + {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -610,8 +610,8 @@ tox-gh-actions = [ {file = "tox_gh_actions-2.9.1-py2.py3-none-any.whl", hash = "sha256:90306a6a04a203e47f861b35dca215fc1f6b7ae80de942a050ace61e2eb2b4ea"}, ] typing-extensions = [ - {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, - {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, + {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, + {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, ] urllib3 = [ {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, From 1eae5b4ced880e061867973293b29c78ea8401e1 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 14:46:17 -0500 Subject: [PATCH 22/27] docs: add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 947aa41f..7b7aaf74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added +* `phylum-init` script entry point and initial functionality * Test workflows for local and CI based testing * Preview and Release workflows for Staging and Production environments * Phylum analyze workflow for PRs From 0815c8eae5c307f36a44eb244f17b668a0be4044 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 20:17:37 -0500 Subject: [PATCH 23/27] fix: account for XDG Base Directory Spec layout for CLI versions > 2.2.0 --- src/phylum/init/cli.py | 104 ++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index ceeb1a41..666533dd 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -45,9 +45,6 @@ } TOKEN_ENVVAR_NAME = "PHYLUM_TOKEN" -PHYLUM_PATH = pathlib.Path.home() / ".phylum" -PHYLUM_BIN_PATH = PHYLUM_PATH / "phylum" -SETTINGS_YAML_PATH = PHYLUM_PATH / "settings.yaml" # Potential features to add: # * Add logging support, a verbosity option to control it, and swap out print statements for logging @@ -58,10 +55,59 @@ # * Add an option to account for pre-releases +def get_phylum_settings_path(version): + """Get the Phylum settings path based on a provided version.""" + home_dir = pathlib.Path.home() + version = version_check(version) + + config_home_path = os.getenv("XDG_CONFIG_HOME") + if not config_home_path: + config_home_path = home_dir / ".config" + phylum_config_path = pathlib.Path(config_home_path) / "phylum" / "settings.yaml" + + # The Phylum config path changed following the v2.2.0 release, to adhere to the XDG Base Directory Spec + # Reference: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + if Version(canonicalize_version(version)) <= Version("v2.2.0"): + phylum_config_path = home_dir / ".phylum" / "settings.yaml" + + return phylum_config_path + + +def get_phylum_bin_path(version): + """Get the path to the Phylum binary based on a provided version.""" + home_dir = pathlib.Path.home() + version = version_check(version) + + # The Phylum binary path changed following the v2.2.0 release, to adhere to the XDG Base Directory Spec + # Reference: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + phylum_bin_path = home_dir / ".local" / "bin" / "phylum" + + if Version(canonicalize_version(version)) <= Version("v2.2.0"): + phylum_bin_path = home_dir / ".phylum" / "phylum" + + return phylum_bin_path + + +def get_latest_version(): + """Get the "latest" version programmatically and return it.""" + # API Reference: https://docs.github.com/en/rest/releases/releases#get-the-latest-release + github_api_url = "https://api.github.com/repos/phylum-dev/cli/releases/latest" + + req = requests.get(github_api_url, timeout=5.0) + req.raise_for_status() + req_json = req.json() + + # The "name" entry stores the GitHub Release name, which could be set to something other than the version. + # Using the "tag_name" entry is better since the tags are much more tightly coupled with the release version. + latest_version = req_json.get("tag_name") + + return latest_version + + def version_check(version): """Check a given version for validity and return a normalized form of it.""" if version == "latest": - return version + version = get_latest_version() version = version.lower() if not version.startswith("v"): @@ -87,7 +133,7 @@ def get_target_triple(): def save_file_from_url(url, path): """Save a file from a given URL to a local file path, in binary mode.""" print(f" [*] Getting {url} file ...", end="") - req = requests.get(url, timeout=2.0) + req = requests.get(url, timeout=5.0) req.raise_for_status() print("Done") @@ -111,16 +157,16 @@ def get_archive_url(version, archive_name): return archive_url -def is_token_set(token=None): - """Check if any token is already set in the CLI configuration file. +def is_token_set(phylum_settings_path, token=None): + """Check if any token is already set in the given CLI configuration file. Optionally, check if a specific given `token` is set. """ - if not SETTINGS_YAML_PATH.exists(): + if not phylum_settings_path.exists(): return False yaml = YAML() - settings_dict = yaml.load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + settings_dict = yaml.load(phylum_settings_path.read_text(encoding="utf-8")) configured_token = settings_dict.get("auth_info", {}).get("offline_access") if configured_token is None: @@ -132,18 +178,20 @@ def is_token_set(token=None): return True -def process_token_option(token_option): +def process_token_option(args): """Process the token option as parsed from the arguments.""" + phylum_settings_path = get_phylum_settings_path(args.version) + # The option takes precedence over the matching environment variable. token = os.getenv(TOKEN_ENVVAR_NAME) - if token_option is not None: - token = token_option + if args.token is not None: + token = args.token if token: print(f" [+] Phylum token supplied as an option or `{TOKEN_ENVVAR_NAME}` environment variable") - if is_token_set(): + if is_token_set(phylum_settings_path): print(" [+] An existing token is already set") - if is_token_set(token=token): + if is_token_set(phylum_settings_path, token=token): print(" [+] Supplied token matches existing token") else: print(" [!] Supplied token will be used to overwrite the existing token") @@ -151,32 +199,35 @@ def process_token_option(token_option): print(" [+] No existing token exists. Supplied token will be used.") else: print(f" [+] Phylum token NOT supplied as option or `{TOKEN_ENVVAR_NAME}` environment variable") - if is_token_set(): + if is_token_set(phylum_settings_path): print(" [+] Existing token found. It will be used without modification.") else: print(" [!] Existing token not found. Use `phylum auth login` or `phylum auth register` command to set it.") - if token and not is_token_set(token=token): - setup_token(token) + if token and not is_token_set(phylum_settings_path, token=token): + setup_token(token, args) + +def setup_token(token, args): + """Setup the CLI credentials with a provided token and path to phylum binary.""" + phylum_bin_path = get_phylum_bin_path(args.version) + phylum_settings_path = get_phylum_settings_path(args.version) -def setup_token(token): - """Setup the CLI credentials with a provided token.""" # The phylum CLI settings.yaml file won't exist upon initial install # but running a command will trigger the CLI to generate it - if not SETTINGS_YAML_PATH.exists(): - cmd_line = [PHYLUM_BIN_PATH, "version"] + if not phylum_settings_path.exists(): + cmd_line = [phylum_bin_path, "version"] subprocess.run(cmd_line, check=True) yaml = YAML() - settings_dict = yaml.load(SETTINGS_YAML_PATH.read_text(encoding="utf-8")) + settings_dict = yaml.load(phylum_settings_path.read_text(encoding="utf-8")) settings_dict.setdefault("auth_info", {}) settings_dict["auth_info"]["offline_access"] = token - with open(SETTINGS_YAML_PATH, "w", encoding="utf-8") as f: + with open(phylum_settings_path, "w", encoding="utf-8") as f: yaml.dump(settings_dict, f) # Check that the token was setup correctly by using it to display the current auth status - cmd_line = [PHYLUM_BIN_PATH, "auth", "status"] + cmd_line = [phylum_bin_path, "auth", "status"] subprocess.run(cmd_line, check=True) @@ -234,6 +285,7 @@ def main(): minisig_name = f"{archive_name}.minisig" archive_url = get_archive_url(args.version, archive_name) minisig_url = f"{archive_url}.minisig" + phylum_bin_path = get_phylum_bin_path(args.version) with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = pathlib.Path(temp_dir) @@ -257,10 +309,10 @@ def main(): cmd_line = ["sh", "install.sh"] subprocess.run(cmd_line, check=True, cwd=extracted_dir) - process_token_option(args.token) + process_token_option(args) # Check to ensure everything is working - cmd_line = [PHYLUM_BIN_PATH, "--help"] + cmd_line = [phylum_bin_path, "--help"] subprocess.run(cmd_line, check=True) return 0 From 370308618278e0c5c52740abbc5917f4df1f520a Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 20:35:05 -0500 Subject: [PATCH 24/27] refactor: remove the `phylum` module shim pointing to `phylum.init` --- README.md | 2 -- src/phylum/__main__.py | 7 ------- tests/functional/test_init.py | 6 +++--- 3 files changed, 3 insertions(+), 12 deletions(-) delete mode 100644 src/phylum/__main__.py diff --git a/README.md b/README.md index b8ed7e2c..456b26b8 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,6 @@ The functionality can also be accessed by calling the module: ```sh python -m phylum.init -h -# The top level package is redirected to the phylum.init package: -python -m phylum -h ``` #### `phylum-init` diff --git a/src/phylum/__main__.py b/src/phylum/__main__.py deleted file mode 100644 index 464187ec..00000000 --- a/src/phylum/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Default to the phylum-init entry point -from phylum.init.cli import main - -# TODO: Add logic here to dynamically show the ways this package can be called as a module. -# Alternate idea: use this as a pass-through to call the true phylum CLI tool. That way, Python can be used to make -# the calls - `python -m phylum` -main() diff --git a/tests/functional/test_init.py b/tests/functional/test_init.py index 45fa2cf1..2b00403b 100644 --- a/tests/functional/test_init.py +++ b/tests/functional/test_init.py @@ -32,8 +32,8 @@ def test_version_option(): """Ensure the correct program name and version is displayed when using the `--version` option.""" # The argparse module adds a newline to the output expected_output = f"{SCRIPT_NAME} {__version__}\n" - cmd_line = [sys.executable, "-m", "phylum", "--version"] - ret = subprocess.run(cmd_line, check=True, capture_output=True, encoding="utf-8") - assert ret.stdout == expected_output, "Output did not match expected input" + cmd_line = [sys.executable, "-m", "phylum.init", "--version"] + ret = subprocess.run(cmd_line, capture_output=True, encoding="utf-8") assert not ret.stderr, "Nothing should be written to stderr" assert ret.returncode == 0, "A non-successful return code was provided" + assert ret.stdout == expected_output, "Output did not match expected input" From 56e7bfb3aefe9eba56e613fcc3643cb66e86ce1b Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 20:36:14 -0500 Subject: [PATCH 25/27] style: remove the list of possible features from comments in code in favor of GitHub issues --- src/phylum/init/cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index 666533dd..5cd9ab0e 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -46,14 +46,6 @@ TOKEN_ENVVAR_NAME = "PHYLUM_TOKEN" -# Potential features to add: -# * Add logging support, a verbosity option to control it, and swap out print statements for logging -# * Check for valid versions by using the GitHub API to compare against actual releases -# * If so, also programmatically generate the SUPPORTED_TARGET_TRIPLES -# * Use the GitHub API ("https://api.github.com") to more programmatically get the archive URL -# * Add a `--list` option, to show which versions are available -# * Add an option to account for pre-releases - def get_phylum_settings_path(version): """Get the Phylum settings path based on a provided version.""" From aad40eba72f1fabc0030c4caeb58d7801f35d617 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Fri, 22 Apr 2022 20:52:19 -0500 Subject: [PATCH 26/27] refactor: update option names, help messages, and zip file extraction logic --- src/phylum/init/cli.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index 5cd9ab0e..e6e7436d 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -232,19 +232,20 @@ def get_args(): ) parser.add_argument( - "-v", - "--phylum-version", + "-r", + "--phylum-release", dest="version", default="latest", type=version_check, - help="the version of the Phylum CLI to install", + help="""The version of the Phylum CLI to install. Can be specified as `latest` or a specific tagged release, + with or without the leading `v`.""", ) parser.add_argument( "-t", "--target", choices=SUPPORTED_TARGET_TRIPLES, default=get_target_triple(), - help="the target platform type where the CLI will be installed", + help="The target platform type where the CLI will be installed.", ) parser.add_argument( "-k", @@ -292,10 +293,7 @@ def main(): with zipfile.ZipFile(archive_path, mode="r") as zip_file: if zip_file.testzip() is not None: raise zipfile.BadZipFile(f"There was a bad file in the zip archive {archive_name}") - extracted_dir = temp_dir_path - top_level_zip_entry = zip_file.infolist()[0] - if top_level_zip_entry.is_dir(): - extracted_dir = temp_dir_path / top_level_zip_entry.filename + extracted_dir = temp_dir_path / f"phylum-{target_triple}" zip_file.extractall(path=temp_dir) cmd_line = ["sh", "install.sh"] From 08cb36f58751874cafed88706f12f930687951f5 Mon Sep 17 00:00:00 2001 From: Charles Coggins Date: Mon, 25 Apr 2022 11:41:03 -0500 Subject: [PATCH 27/27] refactor: respond to review comments --- src/phylum/init/cli.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/phylum/init/cli.py b/src/phylum/init/cli.py index e6e7436d..4e919b4f 100644 --- a/src/phylum/init/cli.py +++ b/src/phylum/init/cli.py @@ -47,6 +47,15 @@ TOKEN_ENVVAR_NAME = "PHYLUM_TOKEN" +def use_legacy_paths(version): + """Predicate to specify whether legacy paths should be used for a given version. + + The Phylum config and binary paths changed following the v2.2.0 release, to adhere to the XDG Base Directory Spec. + Reference: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + """ + return Version(canonicalize_version(version)) <= Version("v2.2.0") + + def get_phylum_settings_path(version): """Get the Phylum settings path based on a provided version.""" home_dir = pathlib.Path.home() @@ -55,11 +64,9 @@ def get_phylum_settings_path(version): config_home_path = os.getenv("XDG_CONFIG_HOME") if not config_home_path: config_home_path = home_dir / ".config" - phylum_config_path = pathlib.Path(config_home_path) / "phylum" / "settings.yaml" - # The Phylum config path changed following the v2.2.0 release, to adhere to the XDG Base Directory Spec - # Reference: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html - if Version(canonicalize_version(version)) <= Version("v2.2.0"): + phylum_config_path = pathlib.Path(config_home_path) / "phylum" / "settings.yaml" + if use_legacy_paths(version): phylum_config_path = home_dir / ".phylum" / "settings.yaml" return phylum_config_path @@ -70,11 +77,8 @@ def get_phylum_bin_path(version): home_dir = pathlib.Path.home() version = version_check(version) - # The Phylum binary path changed following the v2.2.0 release, to adhere to the XDG Base Directory Spec - # Reference: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html phylum_bin_path = home_dir / ".local" / "bin" / "phylum" - - if Version(canonicalize_version(version)) <= Version("v2.2.0"): + if use_legacy_paths(version): phylum_bin_path = home_dir / ".phylum" / "phylum" return phylum_bin_path @@ -136,15 +140,13 @@ def save_file_from_url(url, path): def get_archive_url(version, archive_name): - """Craft an archive download URL from a given version and archive name.""" - github_base_uri = "https://github.com/phylum-dev/cli/releases" - latest_version_uri = f"{github_base_uri}/latest/download" - specific_version_uri = f"{github_base_uri}/download" + """Craft an archive download URL from a given version and archive name. - if version == "latest": - archive_url = f"{latest_version_uri}/{archive_name}" - else: - archive_url = f"{specific_version_uri}/{version}/{archive_name}" + Despite the name, the `version` is really what the GitHub API for releases calls the `tag_name`. + Reference: https://docs.github.com/en/rest/releases/releases#get-a-release-by-tag-name + """ + github_base_uri = "https://github.com/phylum-dev/cli/releases" + archive_url = f"{github_base_uri}/download/{version}/{archive_name}" return archive_url @@ -154,11 +156,13 @@ def is_token_set(phylum_settings_path, token=None): Optionally, check if a specific given `token` is set. """ - if not phylum_settings_path.exists(): + try: + settings_data = phylum_settings_path.read_text(encoding="utf-8") + except FileNotFoundError: return False yaml = YAML() - settings_dict = yaml.load(phylum_settings_path.read_text(encoding="utf-8")) + settings_dict = yaml.load(settings_data) configured_token = settings_dict.get("auth_info", {}).get("offline_access") if configured_token is None: