Generate proper, consistent nightly versions for pip and conda packag…

…es (#1267)

This PR changes conda Python packages and wheels to all generate a consistent version for nightlies. The nightly version is of the form YY.MM.DDaN, where N is the number of commits from the last tag. The version is embedded in both the package metadata and in the `dask_cuda.__version__` attribute. In addition the commit hash itself is embedded into the package as `dask_cuda.__git_commit__`. These changes ensure that

1. wheels are properly considered nightlies and are treated accordingly by pip (e.g. requiring --pre for installation, not conflicting with normal releases, etc)
2. wheels and conda packages are aligned on versions so that they can be easily compared if necessary.

Authors:
  - GALI PREM SAGAR (https://github.com/galipremsagar)

Approvers:
  - Vyas Ramasubramani (https://github.com/vyasr)
  - https://github.com/jakirkham
  - Ray Douglass (https://github.com/raydouglass)

URL: #1267

MANIFEST.in

@@ -1 +1,2 @@
 include dask_cuda/_version.py
+include dask_cuda/VERSION

VERSION

@@ -0,0 +1 @@
+23.12.00

ci/build_python.sh

@@ -9,9 +9,17 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+package_name="dask_cuda"
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+echo "${version}" | tr -d '"' > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py"
+
 rapids-logger "Begin py build"
 
-rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
   conda/recipes/dask-cuda
 
 rapids-upload-conda-to-s3 python

ci/build_python_pypi.sh

@@ -3,6 +3,9 @@
 
 python -m pip install build --user
 
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
 # While conda provides these during conda-build, they are also necessary during
 # the setup.py build for PyPI
 export GIT_DESCRIBE_TAG=$(git describe --abbrev=0 --tags)
@@ -11,25 +14,19 @@ export GIT_DESCRIBE_NUMBER=$(git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count)
 # Build date for PyPI pre-releases using version from `pyproject.toml` as source.
 TOML_VERSION=$(grep "version = .*" pyproject.toml | grep -o '".*"' | sed 's/"//g')
 if ! rapids-is-release-build; then
-  export BUILD_DATE=$(date +%y%m%d)
-  export PACKAGE_VERSION_NUMBER="${TOML_VERSION}a${BUILD_DATE}"
+  export PACKAGE_VERSION_NUMBER="${version}"
 fi
 
+
+echo "${version}" | tr -d '"' > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py"
+
 # Compute/export RAPIDS_DATE_STRING
 source rapids-env-update
 
-# Update pyproject.toml with pre-release build date
-if ! rapids-is-release-build; then
-  sed -i "s/^version = \""${TOML_VERSION}".*\"/version = \""${PACKAGE_VERSION_NUMBER}"\"/g" pyproject.toml
-fi
 
 python -m build \
   --sdist \
   --wheel \
   --outdir dist/ \
   .
-
-# Revert pyproject.toml pre-release build date
-if ! rapids-is-release-build; then
-  sed -i "s/^version = \""${PACKAGE_VERSION_NUMBER}"\"/version = \""${TOML_VERSION}"\"/g" pyproject.toml
-fi

ci/release/update-version.sh

@@ -31,11 +31,8 @@ function sed_runner() {
     sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
 }
 
-# Python __init__.py updates
-sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" dask_cuda/__init__.py
-
-# Python pyproject.toml updates
-sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" pyproject.toml
+# Centralized version file update
+echo "${NEXT_FULL_TAG}" | tr -d '"' > VERSION
 
 # Bump cudf and dask-cudf testing dependencies
 sed_runner "s/cudf=.*/cudf=${NEXT_SHORT_TAG}/g" dependencies.yaml

conda/recipes/dask-cuda/meta.yaml

@@ -4,7 +4,7 @@
 #   conda build -c conda-forge .
 {% set data = load_file_data("pyproject.toml") %}
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].strip('""').lstrip('v') %}
 {% set py_version = environ['CONDA_PY'] %}
 {% set date_string = environ['RAPIDS_DATE_STRING'] %}
 
@@ -13,7 +13,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

dask_cuda/VERSION

@@ -0,0 +1 @@
+../VERSION

dask_cuda/__init__.py

@@ -11,6 +11,7 @@
 import dask.dataframe.multi
 import dask.bag.core
 
+from ._version import __git_commit__, __version__
 from .cuda_worker import CUDAWorker
 from .explicit_comms.dataframe.shuffle import (
     get_rearrange_by_column_wrapper,
@@ -19,7 +20,6 @@
 from .local_cuda_cluster import LocalCUDACluster
 from .proxify_device_objects import proxify_decorator, unproxify_decorator
 
-__version__ = "23.12.00"
 
 # Monkey patching Dask to make use of explicit-comms when `DASK_EXPLICIT_COMMS=True`
 dask.dataframe.shuffle.rearrange_by_column = get_rearrange_by_column_wrapper(

dask_cuda/_version.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+
+__version__ = (
+    importlib.resources.files("dask_cuda").joinpath("VERSION").read_text().strip()
+)
+__git_commit__ = ""

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "dask-cuda"
-version = "23.12.00"
+dynamic = ["version"]
 description = "Utilities for Dask and CUDA interactions"
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [
@@ -123,6 +123,9 @@ filterwarnings = [
 [tool.setuptools]
 license-files = ["LICENSE"]
 
+[tool.setuptools.dynamic]
+version = {file = "dask_cuda/VERSION"}
+
 [tool.setuptools.packages.find]
 exclude = [
     "docs",