diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..a6c342c7 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[run] +branch = True +omit = + */__init__.py + .tox/* + + +[report] +show_missing = true +fail_under = 83 +# due to errors while generating xml report: `NoSource: No source for code` +ignore_errors = True + +[html] +directory = build/coverage diff --git a/hooks/pypi_uploader.py b/hooks/pypi_uploader.py new file mode 100644 index 00000000..f9425de1 --- /dev/null +++ b/hooks/pypi_uploader.py @@ -0,0 +1,244 @@ +"""Conan hook to upload a Conan Python package also as a pip package to a PyPI repository. + +In order to enable this hook do the following: + +* add a new Conan attribute called `pypi` in your Conan recipe and set it to `True` +* set the environment variables `TWINE_USERNAME`, `TWINE_PASSWORD` and `TWINE_REPOSITORY` + to configure `twine` to upload to a PyPI repository. + +The hook is copying the whole exported Conan directory into a temporary folder, creating +a setup.py file, creates a source distribution file and uploads the generated package to +a PyPI repository. +""" + +import json +import os +import shutil +import subprocess +import tempfile + +from urllib.parse import urlparse, urlunsplit +from urllib.request import urlopen +from conans.client import conan_api + +SANDBOX_MODULE_FOUND = False +try: + from setuptools import sandbox + + SANDBOX_MODULE_FOUND = True +except ImportError: + pass + + +def get_setup_py_template(**kwargs): + """Returns the content for a setup.py file based on a template. + + Returns: Content of a setup.py file + """ + return ''' +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""The setup script.""" + +# prevent from normalizing the version in the metadata +from setuptools.extern.packaging import version +version.Version = version.LegacyVersion + +try: + from setuptools import setup, find_packages +except ImportError: + from distutils.core import setup, find_packages + +setup( + description="{description}", + author="conan-io", + author_email="conan@conan.io", + license="", + install_requires=[], + name="{name}", + packages=find_packages(), + version="{version}", + url="{url}", + zip_safe=False, +) +'''.format( + **kwargs + ) + + +def get_recipe_info(conanfile_path): + """Returns recipe information via the Conan API. + + Args: + conanfile_path: Path to conanfile.py + + Returns: recipe info as dictionary + """ + conan_instance, _, _ = conan_api.Conan.factory() + # We need to explicitly request attributes since otherwise it will load only Conan default attributes + # and our custom attribute 'pypi' won't be retrieved. + recipe_attributes = ["name", "url", "description", "pypi"] + return conan_instance.inspect(path=conanfile_path, attributes=recipe_attributes) + + +def _create_setup_py(output, setup_py_path, setup_py_content): + """Creates a setup.py file which will be used later to upload a pip package. + + Args: + output: Conan output object to print formatted messages + setup_py_path: Path to setup.py file + setup_py_content: Content of setup.py file + """ + with open(setup_py_path, "wb") as setup_py_fh: + setup_py_fh.write(setup_py_content.encode("utf-8")) + output.info("Created %s" % setup_py_path) + + +def _create_source_distribution(output, setup_py_path): + """Creates a source distribution by using programmatically setuptools or as an external + python call + + Args: + output: Conan output object to print formatted messages + setup_py_path: Path to setup.py file + """ + output.info("Running `python setup.py sdist`") + if SANDBOX_MODULE_FOUND: + sandbox.run_setup(setup_py_path, ["sdist"]) + else: + out = subprocess.check_output(["python", setup_py_path, "sdist"]) + output.info(out.decode("utf-8")) + + +def _upload_to_pypi(output, pypi_username, pypi_password, pypi_repository): + """Upload the generated source distribution to a PyPI repository + + Args: + output: Conan output object to print formatted messages + pypi_username: The username to authenticate to the PyPI repository + pypi_password: The password to authenticate to the PyPI repository + pypi_repository: The repository to upload the package to + """ + output.info("Uploading to '%s'" % (pypi_repository)) + + # twine does not have an API to be used from within Python, so let's use + # it as an external tool. + out = subprocess.check_output( + [ + "twine", + "upload", + "--verbose", + "-u", + pypi_username, + "-p", + pypi_password, + "--repository-url", + pypi_repository, + "dist/*", + ] + ) + output.info(out.decode("utf-8")) + + +def _is_package_already_uploaded(pypi_repository, package_name, package_version): + """Checks whether Python package is already available in PyPI index. + + Args: + pypi_repository: The repository to search for the package + package_name: Name of the package to be searched for + package_version: Version of the package + + Returns: True if package was already uploaded to the PyPI index otherwise false + """ + parse_result = urlparse(pypi_repository) + if "/artifactory/api" in parse_result.path: + artifactory_api_search_url = urlunsplit( + ( + parse_result.scheme, + parse_result.netloc, + "artifactory/api/search/prop", + "pypi.name=%s&pypi.version=%s" % (package_name, package_version), + "", + ) + ) + response = urlopen(artifactory_api_search_url) + data = json.load(response) + if response.getcode() == 200 and "results" in data and data["results"]: + return True + return False + + +def post_upload(output, conanfile_path, reference, remote, **kwargs): + """[Conan hook](https://docs.conan.io/en/latest/reference/hooks.html) called after whole upload + execution is finished. + + Args: + output: Conan output object to print formatted messages + conanfile_path: Path to the conanfile.py file whether it is in local cache or in user space + reference: Named tuple with attributes name, version, user, and channel + remote: Named tuple with attributes name, url and verify_ssl + """ + # Make pylint happy about unused-arguments linter error. + del remote, kwargs + + recipe_info = get_recipe_info(conanfile_path) + if "pypi" not in recipe_info: + output.info( + "Skipping upload to PyPI repository: 'pypi' attribute not found in Conan project" + ) + return + if not bool(recipe_info["pypi"]): + output.info("Skipping upload to PyPI repository: upload disabled") + return + + twine_settings = ["TWINE_USERNAME", "TWINE_PASSWORD", "TWINE_REPOSITORY"] + if not set(twine_settings).issubset(os.environ): + output.error( + "Missing Twine configuration. Please define the following environment variables: %s" + % twine_settings + ) + return + + # we need to take the version out of the reference since the API returns None only. + version = reference.version + package_name = recipe_info["name"] + template_vars = { + "url": recipe_info["url"], + "description": recipe_info["description"], + "name": package_name, + "version": version, + } + + pypi_repository = os.environ["TWINE_REPOSITORY"] + + if not _is_package_already_uploaded(pypi_repository, package_name, version): + with tempfile.TemporaryDirectory( + prefix="pypi_uploader_%s" % package_name + ) as tmp_dir: + setup_py_dir = os.path.join(tmp_dir, "prj") + shutil.copytree( + src=os.path.dirname(os.path.abspath(conanfile_path)), dst=setup_py_dir + ) + + setup_py_path = os.path.join(setup_py_dir, "setup.py") + setup_py_content = get_setup_py_template(**template_vars) + + os.chdir(setup_py_dir) + + _create_setup_py(output, setup_py_path, setup_py_content) + _create_source_distribution(output, setup_py_path) + _upload_to_pypi( + output, + pypi_username=os.environ["TWINE_USERNAME"], + pypi_password=os.environ["TWINE_PASSWORD"], + pypi_repository=pypi_repository, + ) + output.success( + "Package '%s==%s' uploaded to %s" + % (package_name, version, pypi_repository) + ) + else: + output.warn( + "Package %s==%s is already available in '%s' PyPI index. Upload skipped" + % (package_name, version, pypi_repository) + ) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index eaa8fe04..3a000491 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,3 +1,4 @@ +PyHamcrest==1.9.0 pytest>=3.6 parameterized responses diff --git a/tests/test_hooks/test_pypi_uploader.py b/tests/test_hooks/test_pypi_uploader.py new file mode 100644 index 00000000..b8f4498c --- /dev/null +++ b/tests/test_hooks/test_pypi_uploader.py @@ -0,0 +1,251 @@ +"""Unit tests for pypi_uploader hook. +""" + +import os +import sys + +from unittest.mock import ANY, MagicMock, patch +from hamcrest import contains_string, match_equality +import pytest + +from conans.model.ref import ConanFileReference + +# We don't want to mess up the hooks directory with some __init__.py files and +# therefore add the path to the hooks manually in the test +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from hooks import pypi_uploader + + +def conan_recipe_with_pypi(pypi_attribute_value): + """Returns a conan recipe info dict object with a specific value for the 'pypi' attribute. + + Args: + pypi_attribute_value: Value of 'pypi' attribute (eg. True, False, 0, 1) + + Returns: a recipe info dict object + """ + return { + "name": "abc", + "url": "https://abc.acme.org", + "description": "abc description", + "pypi": pypi_attribute_value, + } + + +def conan_recipe_without_pypi_attr_set(): + """Returns a conan recipe info dict object without the 'pypi' attribute. + + Returns: a recipe info dict object + """ + return { + "name": "abc", + "url": "https://abc.acme.org", + "description": "abc description", + } + + +def with_configured_twine(): + """Set environment variable to configure twine for upload + """ + os.environ["TWINE_REPOSITORY"] = "https://twine.repo" + os.environ["TWINE_USERNAME"] = "user" + os.environ["TWINE_PASSWORD"] = "pwd" + + +@patch("hooks.pypi_uploader.get_recipe_info") +@patch("subprocess.check_output") +def test_should_not_upload_since_pypi_attr_not_set_in_project( + mock_upload_to_pypi_subprocess, mock_get_recipe_info +): + """Expects to print only an info message that the 'pypi' attribute on the Conan project is not set. + """ + # Given + mock_output = MagicMock() + mock_conanfile_path = MagicMock() + mock_reference = MagicMock() + mock_remote = MagicMock() + mock_get_recipe_info.return_value = conan_recipe_without_pypi_attr_set() + + # When + pypi_uploader.post_upload( + output=mock_output, + conanfile_path=mock_conanfile_path, + reference=mock_reference, + remote=mock_remote, + ) + + # Then + mock_output.info.assert_called_with( + "Skipping upload to PyPI repository: 'pypi' attribute not found in Conan project" + ) + mock_upload_to_pypi_subprocess.assert_not_called() + + +@pytest.mark.parametrize("test_input", [False, 0]) +@patch("hooks.pypi_uploader.get_recipe_info") +@patch("subprocess.check_output") +def test_should_not_upload_since_pypi_attr_disabled( + mock_upload_to_pypi_subprocess, mock_get_recipe_info, test_input +): + """Expects to print only an info message that the PyPI upload is disabled. + """ + # Given + mock_output = MagicMock() + mock_conanfile_path = MagicMock() + mock_reference = MagicMock() + mock_remote = MagicMock() + mock_get_recipe_info.return_value = conan_recipe_with_pypi(test_input) + + # When + pypi_uploader.post_upload( + output=mock_output, + conanfile_path=mock_conanfile_path, + reference=mock_reference, + remote=mock_remote, + ) + + # Then + mock_output.info.assert_called_with( + "Skipping upload to PyPI repository: upload disabled" + ) + mock_upload_to_pypi_subprocess.assert_not_called() + + +@patch("hooks.pypi_uploader.get_recipe_info") +@patch("subprocess.check_output") +def test_should_inform_about_missing_twine_configuration( + mock_upload_to_pypi_subprocess, mock_get_recipe_info +): + """Expects to print an error message that twine environment variables are missing. + """ + # Given + mock_output = MagicMock() + mock_conanfile_path = MagicMock() + mock_reference = MagicMock() + mock_remote = MagicMock() + mock_get_recipe_info.return_value = conan_recipe_with_pypi(True) + + # When + pypi_uploader.post_upload( + output=mock_output, + conanfile_path=mock_conanfile_path, + reference=mock_reference, + remote=mock_remote, + ) + + # Then + mock_output.error.assert_called_with( + match_equality( + contains_string( + "Missing Twine configuration. Please define the following environment variables" + ) + ) + ) + mock_upload_to_pypi_subprocess.assert_not_called() + + +@patch("hooks.pypi_uploader.get_recipe_info") +@patch("hooks.pypi_uploader._is_package_already_uploaded") +@patch("subprocess.check_output") +def test_should_inform_that_upload_is_skipped_if_package_is_already_uploaded( + mock_upload_to_pypi_subprocess, mock_is_package_uploaded, mock_get_recipe_info +): + """Expect that upload to PyPI is skipped + """ + # Given + mock_output = MagicMock() + mock_conanfile_path = "/abc/conanfile.py" + + mock_reference = ConanFileReference.loads("abc/1.2.3@ci/stable") + mock_remote = MagicMock() + mock_get_recipe_info.return_value = conan_recipe_with_pypi(True) + mock_is_package_uploaded.return_value = True + + with_configured_twine() + + # When + pypi_uploader.post_upload( + output=mock_output, + conanfile_path=mock_conanfile_path, + reference=mock_reference, + remote=mock_remote, + ) + + # Then + mock_is_package_uploaded.assert_called_with("https://twine.repo", "abc", "1.2.3") + mock_output.warn.assert_called_with( + match_equality(contains_string("Upload skipped")) + ) + mock_upload_to_pypi_subprocess.assert_not_called() + + +@patch("hooks.pypi_uploader.get_recipe_info") +@patch("hooks.pypi_uploader._create_setup_py") +@patch("hooks.pypi_uploader._create_source_distribution") +@patch("hooks.pypi_uploader._is_package_already_uploaded") +@patch("tempfile.TemporaryDirectory") +@patch("shutil.copytree") +@patch("os.chdir") +@patch("subprocess.check_output") +# pylint: disable=too-many-arguments +def test_should_call_twine_to_upload_package_to_pypi_repo( + mock_upload_to_pypi_subprocess, + mock_chdir, + mock_copytree, + mock_tmpdir, + mock_is_package_uploaded, + mock_create_source_dist, + mock_create_setup_py, + mock_get_recipe_info, +): + """Expect that upload to PyPI is not skipped + """ + # Given + mock_output = MagicMock() + mock_conanfile_path = "/abc/conanfile.py" + mock_reference = ConanFileReference.loads("abc/1.2.3@ci/stable") + mock_remote = MagicMock() + + mock_tmpdir.return_value.__enter__.return_value = "/tmp/my_tmp_dir" + # mock_chdir.return_value = "/" + mock_get_recipe_info.return_value = conan_recipe_with_pypi(True) + mock_is_package_uploaded.return_value = False + + with_configured_twine() + + # When + pypi_uploader.post_upload( + output=mock_output, + conanfile_path=mock_conanfile_path, + reference=mock_reference, + remote=mock_remote, + ) + + # Then + mock_copytree.assert_called_with( + src="/abc", + dst=os.path.join(mock_tmpdir.return_value.__enter__.return_value, "prj"), + ) + mock_chdir.assert_called_with("/tmp/my_tmp_dir/prj") + mock_create_setup_py.assert_called_with( + mock_output, "/tmp/my_tmp_dir/prj/setup.py", ANY + ) + mock_create_source_dist.assert_called_with( + mock_output, "/tmp/my_tmp_dir/prj/setup.py" + ) + mock_is_package_uploaded.assert_called_with("https://twine.repo", "abc", "1.2.3") + mock_upload_to_pypi_subprocess.assert_called_with( + [ + "twine", + "upload", + "--verbose", + "-u", + "user", + "-p", + "pwd", + "--repository-url", + "https://twine.repo", + "dist/*", + ] + ) + mock_tmpdir.return_value.__exit__.assert_called_once() diff --git a/tox.ini b/tox.ini index 576c3243..bfd8d5bf 100644 --- a/tox.ini +++ b/tox.ini @@ -16,15 +16,17 @@ deps = conan110: conan>=1.10,<1.11 conan109: conan>=1.9,<1.10 conandev: https://github.com/conan-io/conan/archive/develop.tar.gz - conancurrent: conan + conanlatest: conan + coverage: coverage coverage: coverage-enable-subprocess + coverage: conan -r {toxinidir}/tests/requirements_test.txt setenv = PYTHONDONTWRITEBYTECODE=1 PYTHONPATH = {toxinidir}{:}{env:PYTHONPATH:} - coverage: PYTEST_TEST_RUNNER=coverage run -m pytest + coverage: PYTEST_TEST_RUNNER=coverage run -p -m pytest coverage: COVERAGE_PROCESS_START={toxinidir}/.coveragerc coverage: COVERAGE_FILE={toxinidir}/.coverage coverage: PYTESTDJANGO_COVERAGE_SRC={toxinidir}/ @@ -37,3 +39,22 @@ commands = coverage: coverage combine coverage: coverage report coverage: coverage xml + +[testenv:black] +deps = + black +commands = + black --check --diff {toxinidir}/tests {toxinidir}/hooks + + +[testenv:pylint] +whitelist_externals = + find + bash +skip_install = true +deps = + pyflakes + conan + -r {toxinidir}/tests/requirements_test.txt +commands = + pylint hooks/ tests/