From 40e5a1cc8b6ce739eeb569532b7bbea88bc2acc4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 17 Dec 2024 01:15:05 +0000 Subject: [PATCH 1/8] Set sccache timeout --- .devcontainer/rapids.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 0a85aef3..95f9f0df 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -2,6 +2,7 @@ ARG BASE ARG PYTHON_PACKAGE_MANAGER=conda +ENV SCCACHE_IDLE_TIMEOUT=32768 FROM ${BASE} as pip-base From f96f8ec5af1b620da88dd5f06878292863d649b6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 17 Dec 2024 09:45:44 -0800 Subject: [PATCH 2/8] Update .devcontainer/rapids.Dockerfile Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> --- .devcontainer/rapids.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 95f9f0df..00ab1afb 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -2,7 +2,7 @@ ARG BASE ARG PYTHON_PACKAGE_MANAGER=conda -ENV SCCACHE_IDLE_TIMEOUT=32768 +ENV SCCACHE_IDLE_TIMEOUT=900 FROM ${BASE} as pip-base From 78825be6205f9d56814165c257f8cfdca90c04a3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 17 Dec 2024 20:01:04 +0000 Subject: [PATCH 3/8] Define in each build --- .devcontainer/rapids.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 00ab1afb..1eb7b48e 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -2,7 +2,6 @@ ARG BASE ARG PYTHON_PACKAGE_MANAGER=conda -ENV SCCACHE_IDLE_TIMEOUT=900 FROM ${BASE} as pip-base @@ -31,6 +30,8 @@ ENV DEFAULT_VIRTUAL_ENV=rapids ENV RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY=true +ENV SCCACHE_IDLE_TIMEOUT=900 + FROM ${BASE} as conda-base ENV DEFAULT_CONDA_ENV=rapids @@ -50,6 +51,7 @@ ENV PYTHONDONTWRITEBYTECODE="1" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV SCCACHE_IDLE_TIMEOUT=900 ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV HISTFILE="/home/coder/.cache/._bash_history" From f1721fd69b3ad7bc7cf79e0deb1f745da4870cd1 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Thu, 19 Dec 2024 21:41:10 -0800 Subject: [PATCH 4/8] Update .devcontainer/rapids.Dockerfile --- .devcontainer/rapids.Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 1eb7b48e..dde79282 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -30,8 +30,6 @@ ENV DEFAULT_VIRTUAL_ENV=rapids ENV RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY=true -ENV SCCACHE_IDLE_TIMEOUT=900 - FROM ${BASE} as conda-base ENV DEFAULT_CONDA_ENV=rapids From b55f3668eb10156ff5222bf57cfa66d8d67e0ed8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 24 Dec 2024 11:12:45 -0600 Subject: [PATCH 5/8] try excluding breathe from conda env --- .../src/rapids-build-utils/devcontainer-feature.json | 2 +- .../rapids-build-utils/bin/make-conda-dependencies.sh | 10 +++++----- .../opt/rapids-build-utils/bin/make-conda-env.sh | 3 ++- .../rapids-build-utils/bin/make-pip-dependencies.sh | 10 +++++----- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 3de0e265..ba55907f 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "25.2.3", + "version": "25.2.4", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 9087946b..bc5b0f77 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -9,9 +9,9 @@ # -h,--help Print this text. # # Options that require values: -# -e,--exclude Path(s) to requirement files of packages to exclude. -# -i,--include Path(s) to requirement files of packages to include. -# -k,--key Only include the key(s) +# -e,--exclude Packages to exclude from the final environment (can be provided multiple times). +# -i,--include Packages to include in the final environment (can be provided multiple times). +# -k,--key Only include the key(s) # --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg # of rapids-dependency-file-generator. # (can be passed multiple times) @@ -48,12 +48,12 @@ make_conda_dependencies() { local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do - _exclude+=(-f "${exc}"); + _exclude+=("${exc}"); done local -a _include=(); local inc; for inc in "${include[@]}"; do - _include+=(-f "${inc}"); + _include+=("${inc}"); done local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 0fcb8627..ff5b230b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -35,7 +35,8 @@ make_conda_env() { local -r new_env_path="$(realpath -m "/tmp/${env_file_name}")"; local -r old_env_path="$(realpath -m "${HOME}/.conda/envs/${env_file_name}")"; - rapids-make-conda-dependencies "${OPTS[@]}" > "${new_env_path}"; + # the '--exclude <(echo breathe)' can be removed when https://github.com/conda-forge/breathe-feedstock/pull/64 is merged + rapids-make-conda-dependencies --exclude <(echo breathe) "${OPTS[@]}" > "${new_env_path}"; if test -f "${new_env_path}" && test "$(wc -l "${new_env_path}" | cut -d' ' -f1)" -gt 0; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 4a1ec400..c5df0de9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Usage: -# rapids-make-conda-dependencies [OPTION]... +# rapids-make-pip-dependencies [OPTION]... # # Generate a combined pip requirements file for all repos. # @@ -10,8 +10,8 @@ # --no-dedupe Don't sort and dedupe the combined requirements.txt. # # Options that require values: -# -e,--exclude Path(s) to requirement files of packages to exclude. -# -i,--include Path(s) to requirement files of packages to include. +# -e,--exclude Packages to exclude from the final environment (can be provided multiple times). +# -i,--include Packages to include in the final environment (can be provided multiple times). # -k,--key Only include the key(s) # --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg # of rapids-dependency-file-generator. @@ -49,12 +49,12 @@ make_pip_dependencies() { local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do - _exclude+=(-f "${exc}"); + _exclude+=("${exc}"); done local -a _include=(); local inc; for inc in "${include[@]}"; do - _include+=(-f "${inc}"); + _include+=("${inc}"); done local cuda_version="${CUDA_VERSION_MAJOR_MINOR:-}"; From 7f93231a9510258ec4b93e05e01e266d489d281e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 24 Dec 2024 11:45:17 -0600 Subject: [PATCH 6/8] revert incorrect changes, expand comments and docs --- .../bin/make-conda-dependencies.sh | 14 +++++++++----- .../bin/make-pip-dependencies.sh | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index bc5b0f77..4e4124fd 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -9,9 +9,11 @@ # -h,--help Print this text. # # Options that require values: -# -e,--exclude Packages to exclude from the final environment (can be provided multiple times). -# -i,--include Packages to include in the final environment (can be provided multiple times). -# -k,--key Only include the key(s) +# -e,--exclude Path(s) to requirement files of packages to exclude. +# Can also be a file descriptor like '<(echo libucx)'. +# -i,--include Path(s) to requirement files of packages to include. +# Can also be a file descriptor like '<(echo libucx)'. +# -k,--key Only include the key(s) # --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg # of rapids-dependency-file-generator. # (can be passed multiple times) @@ -48,12 +50,14 @@ make_conda_dependencies() { local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do - _exclude+=("${exc}"); + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _exclude+=(-f "${exc}"); done local -a _include=(); local inc; for inc in "${include[@]}"; do - _include+=("${inc}"); + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _include+=(-f "${inc}"); done local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index c5df0de9..c3a198f2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -10,9 +10,11 @@ # --no-dedupe Don't sort and dedupe the combined requirements.txt. # # Options that require values: -# -e,--exclude Packages to exclude from the final environment (can be provided multiple times). -# -i,--include Packages to include in the final environment (can be provided multiple times). -# -k,--key Only include the key(s) +# -e,--exclude Path(s) to requirement files of packages to exclude. +# Can also be a file descriptor like '<(echo libucx)'. +# -i,--include Path(s) to requirement files of packages to include. +# Can also be a file descriptor like '<(echo libucx)'. +# -k,--key Only include the key(s) # --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg # of rapids-dependency-file-generator. # (can be passed multiple times) @@ -49,12 +51,14 @@ make_pip_dependencies() { local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do - _exclude+=("${exc}"); + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _exclude+=(-f "${exc}"); done local -a _include=(); local inc; for inc in "${include[@]}"; do - _include+=("${inc}"); + # append '-f' so each file's contents will be treated as a list of patterns for 'grep' + _include+=(-f "${inc}"); done local cuda_version="${CUDA_VERSION_MAJOR_MINOR:-}"; From a207cf98c1c4c77c2c6ab317ef1731c1494a962c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 24 Dec 2024 11:47:19 -0600 Subject: [PATCH 7/8] revert unintentional whitespace changes --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index c3a198f2..c3e9649c 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -14,7 +14,7 @@ # Can also be a file descriptor like '<(echo libucx)'. # -i,--include Path(s) to requirement files of packages to include. # Can also be a file descriptor like '<(echo libucx)'. -# -k,--key Only include the key(s) +# -k,--key Only include the key(s) # --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg # of rapids-dependency-file-generator. # (can be passed multiple times) From 55e2a236f945be78f9ede232577565774da953f0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 30 Dec 2024 20:25:12 +0000 Subject: [PATCH 8/8] Remove Breathe special-casing --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index ff5b230b..0fcb8627 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -35,8 +35,7 @@ make_conda_env() { local -r new_env_path="$(realpath -m "/tmp/${env_file_name}")"; local -r old_env_path="$(realpath -m "${HOME}/.conda/envs/${env_file_name}")"; - # the '--exclude <(echo breathe)' can be removed when https://github.com/conda-forge/breathe-feedstock/pull/64 is merged - rapids-make-conda-dependencies --exclude <(echo breathe) "${OPTS[@]}" > "${new_env_path}"; + rapids-make-conda-dependencies "${OPTS[@]}" > "${new_env_path}"; if test -f "${new_env_path}" && test "$(wc -l "${new_env_path}" | cut -d' ' -f1)" -gt 0; then