From 4b4c02136f54d5a79a8b009626baf6df69656d69 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Fri, 2 Dec 2022 15:14:53 -0500 Subject: [PATCH 1/6] docs for making release in brew and pypi --- developer_docs/cromshell_package_releases.md | 91 ++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 developer_docs/cromshell_package_releases.md diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md new file mode 100644 index 00000000..ac56a7d3 --- /dev/null +++ b/developer_docs/cromshell_package_releases.md @@ -0,0 +1,91 @@ +# Cromshell Package Releases + +Cromshell can be installed from a brew tap or through pypi. Having these sources +updated to have the latest releases of Cromshell is important. + +## Brew Tap + +Regular update of broadinstitute/dsp tap is needed as new releases of Cromshell are made. +Once a release occurs a new formula for that release should be created in +[broadinstitute/homebrew-dsp](https://github.com/broadinstitute/homebrew-dsp) git repository +in the Formula directory. + +The name of the formula file should have the following structure: +`cromshell@[version].rb`. For example: + + cromshell@1.0.3.rb + + +The following should be updated regularly in a new formula file: +- Git release tag url +- Git release tag hash +- List of required python package url and their hash + +Here is an example of the content of the Formula file. +```Ruby +class CromshellAT200Alpha1 < Formula + include Language::Python::Virtualenv + + desc "Python tool to interact with a Cromwell server" + homepage "https://github.com/broadinstitute/cromshell" + url "https://github.com/broadinstitute/cromshell/archive/refs/tags/2.0.0.alpha.1.tar.gz" + sha256 "0ac3604ff362fdf2107f91c886887856098f1731d2d1a9e9cff59278b7b292b9" + license "BSD-3-Clause" + depends_on "python@3.9" + + resource "termcolor" do + url "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz" + sha256 "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" + end + resource "click>=8.0.0" do + url "https://files.pythonhosted.org/packages/f4/09/ad003f1e3428017d1c3da4ccc9547591703ffea548626f47ec74509c5824/click-8.0.3.tar.gz" + sha256 "410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + end + resource "pygments" do + url "https://files.pythonhosted.org/packages/b7/b3/5cba26637fe43500d4568d0ee7b7362de1fb29c0e158d50b4b69e9a40422/Pygments-2.10.0.tar.gz" + sha256 "f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6" + end + # below are dependencies of requests + resource "urllib3" do + url "https://files.pythonhosted.org/packages/80/be/3ee43b6c5757cabea19e75b8f46eaf05a2f5144107d7db48c7cf3a864f73/urllib3-1.26.7.tar.gz" + sha256 "4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece" + end + resource "certifi" do + url "https://files.pythonhosted.org/packages/6c/ae/d26450834f0acc9e3d1f74508da6df1551ceab6c2ce0766a593362d6d57f/certifi-2021.10.8.tar.gz" + sha256 "78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872" + end + resource "charset-normalizer" do + url "https://files.pythonhosted.org/packages/9f/c5/334c019f92c26e59637bb42bd14a190428874b2b2de75a355da394cf16c1/charset-normalizer-2.0.7.tar.gz" + sha256 "e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0" + end + resource "idna" do + url "https://files.pythonhosted.org/packages/62/08/e3fc7c8161090f742f504f40b1bccbfc544d4a4e09eb774bf40aafce5436/idna-3.3.tar.gz" + sha256 "9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + end + resource "requests" do + url "https://files.pythonhosted.org/packages/e7/01/3569e0b535fb2e4a6c384bdbed00c55b9d78b5084e0fb7f4d0bf523d7670/requests-2.26.0.tar.gz" + sha256 "b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + end + + def install + virtualenv_install_with_resources + end + + # `test do` will create, run in, and delete a temporary directory. + test do + system "#{bin}/cromshell-alpha", "--help" + end +end +``` + + +## Pypi +Currently, Cromshell is available on testpypi, later it will be made available on +the main pypi site. + +A git action is available to help easily send a release to pypi. The action needs to be +executed manually with `PyPI_Repo` (by default set to "testpypi" for now) and `version` inputs. + +Once a release version is created in the pypi repository, it is not possible rewrite over it +(like in the case of a bug fix) so it's best to allow a release to have in the 'wild' for +testing before adding the release to pypi. From 508bf57115507c8cfba26aed8a80293bccd24adc Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 15 Dec 2022 11:24:44 -0500 Subject: [PATCH 2/6] Added steps to publishing to pypi and brew --- developer_docs/cromshell_package_releases.md | 87 +++++++++++++------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md index ac56a7d3..11c07d1a 100644 --- a/developer_docs/cromshell_package_releases.md +++ b/developer_docs/cromshell_package_releases.md @@ -1,7 +1,6 @@ # Cromshell Package Releases -Cromshell can be installed from a brew tap or through pypi. Having these sources -updated to have the latest releases of Cromshell is important. +Cromshell can be installed from a brew tap or through pypi. ## Brew Tap @@ -41,31 +40,6 @@ class CromshellAT200Alpha1 < Formula url "https://files.pythonhosted.org/packages/f4/09/ad003f1e3428017d1c3da4ccc9547591703ffea548626f47ec74509c5824/click-8.0.3.tar.gz" sha256 "410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" end - resource "pygments" do - url "https://files.pythonhosted.org/packages/b7/b3/5cba26637fe43500d4568d0ee7b7362de1fb29c0e158d50b4b69e9a40422/Pygments-2.10.0.tar.gz" - sha256 "f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6" - end - # below are dependencies of requests - resource "urllib3" do - url "https://files.pythonhosted.org/packages/80/be/3ee43b6c5757cabea19e75b8f46eaf05a2f5144107d7db48c7cf3a864f73/urllib3-1.26.7.tar.gz" - sha256 "4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece" - end - resource "certifi" do - url "https://files.pythonhosted.org/packages/6c/ae/d26450834f0acc9e3d1f74508da6df1551ceab6c2ce0766a593362d6d57f/certifi-2021.10.8.tar.gz" - sha256 "78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872" - end - resource "charset-normalizer" do - url "https://files.pythonhosted.org/packages/9f/c5/334c019f92c26e59637bb42bd14a190428874b2b2de75a355da394cf16c1/charset-normalizer-2.0.7.tar.gz" - sha256 "e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0" - end - resource "idna" do - url "https://files.pythonhosted.org/packages/62/08/e3fc7c8161090f742f504f40b1bccbfc544d4a4e09eb774bf40aafce5436/idna-3.3.tar.gz" - sha256 "9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" - end - resource "requests" do - url "https://files.pythonhosted.org/packages/e7/01/3569e0b535fb2e4a6c384bdbed00c55b9d78b5084e0fb7f4d0bf523d7670/requests-2.26.0.tar.gz" - sha256 "b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" - end def install virtualenv_install_with_resources @@ -78,12 +52,65 @@ class CromshellAT200Alpha1 < Formula end ``` +Note: +1. All packages listed in the requirements.txt file should be listed in the formula. +Sometimes packages in the file have their own dependencies, you can identify this +by using a tool like `pipdeptree` to list whether it has dependencies. + ```shell + >pipdeptree -p requests + requests==2.27.1 + - certifi [required: >=2017.4.17, installed: 2020.12.5] + - charset-normalizer [required: ~=2.0.0, installed: 2.0.12] + - idna [required: >=2.5,<4, installed: 2.10] + - urllib3 [required: >=1.21.1,<1.27, installed: 1.26.2] + ``` + +2. The package url and hash can be obtained from the package pypi site or using a tool like +python hashin. + + ``` + import hashin + hashin.get_package_hashes('requests', version='2.27.1') + ``` + + results to: + + ``` + {'package': 'requests', 'version': '2.27.1', 'hashes': [{'hash': '68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61'}, {'hash': 'f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d'}]} + ``` +### Test Install + + ``` + brew tap broadinstitute/dsp + brew install cromshell + ``` ## Pypi -Currently, Cromshell is available on testpypi, later it will be made available on -the main pypi site. +Use steps below to release on either PyPi or testPyPi, the latter is primarily used for +draft releases. + +1. Install the following tools + + `python3 -m pip install --upgrade pip twine build` + +2. In the root repository directory build cromshell + + `python3 -m build` + +3. Update the version tag and project name in pyproject.toml. If creating a release in +`testPyPi` use project name `cromshell-draft-release`, if creating a release in `PyPi` +use project name `cromshell`. + +4. Publish build using twine. The [PYPI Token](https://pypi.org/help/#apitoken) is obtained from your pypi account, your + account will need access permission to the cromwell project in pypi. + + `python3 -m twine upload --username __token__ --password --repository pypi dist/*` + +### Pypi git action [Not Available] +*Because this git yml is only initiated through `workflow dispatch`, the action will not be +functional until the `cromshell2` branch is merged to `main`.* -A git action is available to help easily send a release to pypi. The action needs to be +A git action has been created to help easily send a release to pypi. The action needs to be executed manually with `PyPI_Repo` (by default set to "testpypi" for now) and `version` inputs. Once a release version is created in the pypi repository, it is not possible rewrite over it From 9e8d4006fafe25260ac84971168edc5b287b0293 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Wed, 4 Jan 2023 14:35:13 -0500 Subject: [PATCH 3/6] added commands to release docs --- developer_docs/cromshell_package_releases.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md index 11c07d1a..7b9c4717 100644 --- a/developer_docs/cromshell_package_releases.md +++ b/developer_docs/cromshell_package_releases.md @@ -16,8 +16,8 @@ The name of the formula file should have the following structure: The following should be updated regularly in a new formula file: -- Git release tag url -- Git release tag hash +- Git release tar file url +- Git release tar file hash (e.g. `shasum -a 256 /path/to/tar/cromshell-2.0.0.beta.1.tar.gz`) - List of required python package url and their hash Here is an example of the content of the Formula file. From 1a4d770dace15e86283acfe482945b802e043b71 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 5 Jan 2023 07:27:27 -0500 Subject: [PATCH 4/6] added homebrew-pypi poet tool to use in the release doc --- developer_docs/cromshell_package_releases.md | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md index 7b9c4717..31a06fe4 100644 --- a/developer_docs/cromshell_package_releases.md +++ b/developer_docs/cromshell_package_releases.md @@ -66,20 +66,34 @@ by using a tool like `pipdeptree` to list whether it has dependencies. ``` 2. The package url and hash can be obtained from the package pypi site or using a tool like -python hashin. +homebrew-pypi-poet. ``` - import hashin - hashin.get_package_hashes('requests', version='2.27.1') + poet requests ``` results to: ``` - {'package': 'requests', 'version': '2.27.1', 'hashes': [{'hash': '68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61'}, {'hash': 'f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d'}]} + resource "certifi" do + url "https://files.pythonhosted.org/packages/6c/ae/d26450834f0acc9e3d1f74508da6df1551ceab6c2ce0766a593362d6d57f/certifi-2021.10.8.tar.gz" + sha256 "78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872" + end + + resource "charset-normalizer" do + url "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz" + sha256 "2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597" + end + ... ``` ### Test Install +Test installation using formula. + ``` + brew install --build-from-source cromshell@2.0.0.beta.1.rb + ``` + +Test installation using brew tap ``` brew tap broadinstitute/dsp brew install cromshell From 6053f9ed5435a19b7b4e144087053c2c9a200cd0 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 5 Jan 2023 08:06:51 -0500 Subject: [PATCH 5/6] corrected steps to create publish to pypi --- developer_docs/cromshell_package_releases.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md index 31a06fe4..48a65249 100644 --- a/developer_docs/cromshell_package_releases.md +++ b/developer_docs/cromshell_package_releases.md @@ -107,18 +107,18 @@ draft releases. `python3 -m pip install --upgrade pip twine build` -2. In the root repository directory build cromshell - - `python3 -m build` - -3. Update the version tag and project name in pyproject.toml. If creating a release in +2. Update the version tag and project name in pyproject.toml. If creating a release in `testPyPi` use project name `cromshell-draft-release`, if creating a release in `PyPi` use project name `cromshell`. + +3. In the root repository directory build cromshell. This will create a folder called `/dist` with tar files of the cromshell build. + + `python3 -m build` 4. Publish build using twine. The [PYPI Token](https://pypi.org/help/#apitoken) is obtained from your pypi account, your - account will need access permission to the cromwell project in pypi. + account will need access permission to the cromwell project in pypi (you will have different tokens for testpypi and pypi repos). - `python3 -m twine upload --username __token__ --password --repository pypi dist/*` + `python3 -m twine upload --username __token__ --password --repository dist/*` ### Pypi git action [Not Available] *Because this git yml is only initiated through `workflow dispatch`, the action will not be From 5fde0f3a9ede1c2c4e23752a3331727e94bcd354 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Thu, 5 Jan 2023 08:16:06 -0500 Subject: [PATCH 6/6] minor edits --- developer_docs/cromshell_package_releases.md | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/developer_docs/cromshell_package_releases.md b/developer_docs/cromshell_package_releases.md index 48a65249..7502b2a0 100644 --- a/developer_docs/cromshell_package_releases.md +++ b/developer_docs/cromshell_package_releases.md @@ -1,6 +1,6 @@ # Cromshell Package Releases -Cromshell can be installed from a brew tap or through pypi. +Cromshell can be installed from a brew tap or through pypi. Instructions on how to publish to these repos are detailed below. ## Brew Tap @@ -75,16 +75,30 @@ homebrew-pypi-poet. results to: ``` - resource "certifi" do - url "https://files.pythonhosted.org/packages/6c/ae/d26450834f0acc9e3d1f74508da6df1551ceab6c2ce0766a593362d6d57f/certifi-2021.10.8.tar.gz" - sha256 "78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872" + resource "certifi" do + url "https://files.pythonhosted.org/packages/06/a9/cd1fd8ee13f73a4d4f491ee219deeeae20afefa914dfb4c130cfc9dc397a/certifi-2020.12.5.tar.gz" + sha256 "1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c" end resource "charset-normalizer" do url "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz" sha256 "2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597" end - ... + + resource "idna" do + url "https://files.pythonhosted.org/packages/ea/b7/e0e3c1c467636186c39925827be42f16fee389dc404ac29e930e9136be70/idna-2.10.tar.gz" + sha256 "b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6" + end + + resource "requests" do + url "https://files.pythonhosted.org/packages/60/f3/26ff3767f099b73e0efa138a9998da67890793bfa475d8278f84a30fec77/requests-2.27.1.tar.gz" + sha256 "68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61" + end + + resource "urllib3" do + url "https://files.pythonhosted.org/packages/29/e6/d1a1d78c439cad688757b70f26c50a53332167c364edb0134cadd280e234/urllib3-1.26.2.tar.gz" + sha256 "19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08" + end ``` ### Test Install