From d9e10013afb5501f1438d2a7f8d0394368e30108 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 27 Oct 2023 17:03:32 -0500 Subject: [PATCH] Generate proper, consistent nightly versions for pip and conda packages (#1267) This PR changes conda Python packages and wheels to all generate a consistent version for nightlies. The nightly version is of the form YY.MM.DDaN, where N is the number of commits from the last tag. The version is embedded in both the package metadata and in the `dask_cuda.__version__` attribute. In addition the commit hash itself is embedded into the package as `dask_cuda.__git_commit__`. These changes ensure that 1. wheels are properly considered nightlies and are treated accordingly by pip (e.g. requiring --pre for installation, not conflicting with normal releases, etc) 2. wheels and conda packages are aligned on versions so that they can be easily compared if necessary. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - https://github.com/jakirkham - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/dask-cuda/pull/1267 --- MANIFEST.in | 1 + VERSION | 1 + ci/build_python.sh | 10 +++++++++- ci/build_python_pypi.sh | 19 ++++++++----------- ci/release/update-version.sh | 7 ++----- conda/recipes/dask-cuda/meta.yaml | 4 ++-- dask_cuda/VERSION | 1 + dask_cuda/__init__.py | 2 +- dask_cuda/_version.py | 20 ++++++++++++++++++++ pyproject.toml | 5 ++++- 10 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 VERSION create mode 120000 dask_cuda/VERSION create mode 100644 dask_cuda/_version.py diff --git a/MANIFEST.in b/MANIFEST.in index 344d51cc8..d97770d06 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include dask_cuda/_version.py +include dask_cuda/VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..a193fff41 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +23.12.00 diff --git a/ci/build_python.sh b/ci/build_python.sh index d4a28497d..23c806704 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,9 +9,17 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +package_name="dask_cuda" + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) + +echo "${version}" | tr -d '"' > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py" + rapids-logger "Begin py build" -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ conda/recipes/dask-cuda rapids-upload-conda-to-s3 python diff --git a/ci/build_python_pypi.sh b/ci/build_python_pypi.sh index 6b72b96d7..b13783d16 100755 --- a/ci/build_python_pypi.sh +++ b/ci/build_python_pypi.sh @@ -3,6 +3,9 @@ python -m pip install build --user + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) # While conda provides these during conda-build, they are also necessary during # the setup.py build for PyPI export GIT_DESCRIBE_TAG=$(git describe --abbrev=0 --tags) @@ -11,25 +14,19 @@ export GIT_DESCRIBE_NUMBER=$(git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count) # Build date for PyPI pre-releases using version from `pyproject.toml` as source. TOML_VERSION=$(grep "version = .*" pyproject.toml | grep -o '".*"' | sed 's/"//g') if ! rapids-is-release-build; then - export BUILD_DATE=$(date +%y%m%d) - export PACKAGE_VERSION_NUMBER="${TOML_VERSION}a${BUILD_DATE}" + export PACKAGE_VERSION_NUMBER="${version}" fi + +echo "${version}" | tr -d '"' > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py" + # Compute/export RAPIDS_DATE_STRING source rapids-env-update -# Update pyproject.toml with pre-release build date -if ! rapids-is-release-build; then - sed -i "s/^version = \""${TOML_VERSION}".*\"/version = \""${PACKAGE_VERSION_NUMBER}"\"/g" pyproject.toml -fi python -m build \ --sdist \ --wheel \ --outdir dist/ \ . - -# Revert pyproject.toml pre-release build date -if ! rapids-is-release-build; then - sed -i "s/^version = \""${PACKAGE_VERSION_NUMBER}"\"/version = \""${TOML_VERSION}"\"/g" pyproject.toml -fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index c0e8c11d2..94cd5d12b 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -31,11 +31,8 @@ function sed_runner() { sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak } -# Python __init__.py updates -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" dask_cuda/__init__.py - -# Python pyproject.toml updates -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" pyproject.toml +# Centralized version file update +echo "${NEXT_FULL_TAG}" | tr -d '"' > VERSION # Bump cudf and dask-cudf testing dependencies sed_runner "s/cudf=.*/cudf=${NEXT_SHORT_TAG}/g" dependencies.yaml diff --git a/conda/recipes/dask-cuda/meta.yaml b/conda/recipes/dask-cuda/meta.yaml index 3b0c15626..6804b1ce4 100644 --- a/conda/recipes/dask-cuda/meta.yaml +++ b/conda/recipes/dask-cuda/meta.yaml @@ -4,7 +4,7 @@ # conda build -c conda-forge . {% set data = load_file_data("pyproject.toml") %} -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].strip('""').lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -13,7 +13,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/dask_cuda/VERSION b/dask_cuda/VERSION new file mode 120000 index 000000000..6ff19de4b --- /dev/null +++ b/dask_cuda/VERSION @@ -0,0 +1 @@ +../VERSION \ No newline at end of file diff --git a/dask_cuda/__init__.py b/dask_cuda/__init__.py index 9d6917ef6..dbbb1f7fb 100644 --- a/dask_cuda/__init__.py +++ b/dask_cuda/__init__.py @@ -11,6 +11,7 @@ import dask.dataframe.multi import dask.bag.core +from ._version import __git_commit__, __version__ from .cuda_worker import CUDAWorker from .explicit_comms.dataframe.shuffle import ( get_rearrange_by_column_wrapper, @@ -19,7 +20,6 @@ from .local_cuda_cluster import LocalCUDACluster from .proxify_device_objects import proxify_decorator, unproxify_decorator -__version__ = "23.12.00" # Monkey patching Dask to make use of explicit-comms when `DASK_EXPLICIT_COMMS=True` dask.dataframe.shuffle.rearrange_by_column = get_rearrange_by_column_wrapper( diff --git a/dask_cuda/_version.py b/dask_cuda/_version.py new file mode 100644 index 000000000..c54072ba5 --- /dev/null +++ b/dask_cuda/_version.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files("dask_cuda").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/pyproject.toml b/pyproject.toml index 2ebe09bc7..c240e61b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ [project] name = "dask-cuda" -version = "23.12.00" +dynamic = ["version"] description = "Utilities for Dask and CUDA interactions" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -123,6 +123,9 @@ filterwarnings = [ [tool.setuptools] license-files = ["LICENSE"] +[tool.setuptools.dynamic] +version = {file = "dask_cuda/VERSION"} + [tool.setuptools.packages.find] exclude = [ "docs",