From 0dd18bdcd4837e918cbf753fb2b7c3f5dcef066f Mon Sep 17 00:00:00 2001 From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:59:20 -0700 Subject: [PATCH] gromacs (#197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create gromacs/application.py * Add gromacs openmp config (#62) * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll * copyright * copyright * copyright * Draft ADAC experiment setup * CUDA experiment for LLNL-Sierra-IBM-power9-V100-Infiniband * CUDA config for LLNL-Pascal-Penguin-broadwell-P100-OmniPat * Fix target name * CUDA config for lassen * Fix cuda config * Set Openmp thread affinity to cores * Fix cuda compiler for gromacs * gromacs rocm config * gromacs rocm config * Remove spack_setup * ROCm packages: externals consistency (#150) * remove version locking from experiment config; achieve this with a mixin package * add needed externals, proper dependency reference for mixin * also align rocmcc version w/hip version * Spack package from Loic Pottier * Remove duplicate config entry * Fix gromacs configs * gromacs config changes * Fix gromacs openmp config * Fix license --------- Co-authored-by: pearce8 Co-authored-by: Szilárd Páll Co-authored-by: Riyaz Haque Co-authored-by: Peter Scheibel --- .../auxiliary_software_files/compilers.yaml | 46 ++++++-- .../auxiliary_software_files/packages.yaml | 67 +++++++++++- .../spack.yaml | 14 +-- experiments/gromacs/openmp/ramble.yaml | 2 +- experiments/gromacs/rocm/ramble.yaml | 12 +-- repo/gromacs/package.py | 72 +++++++++---- repo/hipsycl/package.py | 102 ++++++++++-------- repo/rocm-consistency/package.py | 41 +++++++ 8 files changed, 270 insertions(+), 86 deletions(-) create mode 100644 repo/rocm-consistency/package.py diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml index d0ded907e..173623d72 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml @@ -23,21 +23,55 @@ compilers: LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/] - compiler: - spec: clang@16.0.0-rocm5.5.1 + spec: rocmcc@5.5.1 paths: cc: /opt/rocm-5.5.1/bin/amdclang cxx: /opt/rocm-5.5.1/bin/amdclang++ f77: /opt/rocm-5.5.1/bin/amdflang fc: /opt/rocm-5.5.1/bin/amdflang flags: - cflags: -g -O2 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr - cxxflags: -g -O2 -std=c++17 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr - fflags: -g -O2 -hnopattern + cflags: -g -O2 + cxxflags: -g -O2 operating_system: rhel8 target: x86_64 modules: [] - environment: {} - extra_rpaths: [] + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs + prepend_path: + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib" + LIBRARY_PATH: /opt/rocm-5.5.1/lib + extra_rpaths: + - /opt/rocm-5.5.1/lib + - /opt/cray/pe/gcc-libs + - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib +- compiler: + spec: rocmcc@5.4.3 + paths: + cc: /opt/rocm-5.4.3/bin/amdclang + cxx: /opt/rocm-5.4.3/bin/amdclang++ + f77: /opt/rocm-5.4.3/bin/amdflang + fc: /opt/rocm-5.4.3/bin/amdflang + flags: + cflags: -g -O2 + cxxflags: -g -O2 + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs + prepend_path: + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib" + LIBRARY_PATH: /opt/rocm-5.4.3/lib + extra_rpaths: + - /opt/rocm-5.4.3/lib + - /opt/cray/pe/gcc-libs + - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib - compiler: spec: gcc@12.2.0 paths: diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml index 35e5ee517..c13e93f75 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml @@ -183,6 +183,55 @@ packages: - spec: hsa-rocr-dev@5.5.1 prefix: /opt/rocm-5.5.1 buildable: false + comgr: + buildable: false + externals: + - spec: comgr@5.4.3 + prefix: /opt/rocm-5.4.3/ + - spec: comgr@5.5.1 + prefix: /opt/rocm-5.5.1/ + hipsparse: + buildable: false + externals: + - spec: hipsparse@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipsparse@5.5.1 + prefix: /opt/rocm-5.5.1 + hipblas: + buildable: false + externals: + - spec: hipblas@5.4.3 + prefix: /opt/rocm-5.4.3/ + - spec: hipblas@5.5.1 + prefix: /opt/rocm-5.5.1/ + hsakmt-roct: + buildable: false + externals: + - spec: hsakmt-roct@5.4.3 + prefix: /opt/rocm-5.4.3/ + - spec: hsakmt-roct@5.5.1 + prefix: /opt/rocm-5.5.1/ + roctracer-dev-api: + buildable: false + externals: + - spec: roctracer-dev-api@5.4.3 + prefix: /opt/rocm-5.4.3/ + - spec: roctracer-dev-api@5.5.1 + prefix: /opt/rocm-5.5.1/ + rocminfo: + buildable: false + externals: + - spec: rocminfo@5.4.3 + prefix: /opt/rocm-5.4.3/ + - spec: rocminfo@5.5.1 + prefix: /opt/rocm-5.5.1/ + llvm: + buildable: false + externals: + - spec: llvm@15.0.0-5.4.3 + prefix: /opt/rocm-5.4.3/llvm + - spec: llvm@16.0.0-5.5.1 + prefix: /opt/rocm-5.5.1/llvm llvm-amdgpu: externals: - spec: llvm-amdgpu@5.4.3 @@ -215,6 +264,13 @@ packages: prefix: /opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/ lapack: buildable: false + blas: + buildable: false + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr hypre: variants: amdgpu_target=gfx90a hwloc: @@ -231,22 +287,27 @@ packages: buildable: false mpi: buildable: false + libfabric: + externals: + - spec: libfabric@2.1 + prefix: /opt/cray/libfabric/2.1 + buildable: false cray-mpich: buildable: false externals: - - spec: cray-mpich@8.1.26%cce +gtl +wrappers + - spec: cray-mpich@8.1.26%cce@16.0.0 +gtl +wrappers prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 extra_attributes: gtl_cutoff_size: 4096 fi_cxi_ats: 0 gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa" - - spec: cray-mpich@8.1.26%cce ~gtl +wrappers + - spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl +wrappers prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 extra_attributes: gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib" - - spec: cray-mpich@8.1.26%gcc ~gtl +wrappers + - spec: cray-mpich@8.1.26%gcc@12.2.0 ~gtl +wrappers prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3 extra_attributes: gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml index 7f13f5d53..f1b5fb731 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml @@ -6,28 +6,28 @@ spack: packages: default-compiler: - spack_spec: cce@16 + spack_spec: cce@16.0.0-rocm5.5.1 default-mpi: spack_spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl compiler-rocm: - spack_spec: cce@16 + spack_spec: cce@16.0.0-rocm5.5.1 compiler-amdclang: spack_spec: clang@16.0.0-rocm5.5.1 compiler-gcc: spack_spec: gcc@12.2.0 blas-rocm: - spack_spec: rocblas@5.5 + spack_spec: rocblas@5.5.1 blas: - spack_spec: rocblas@5.5 + spack_spec: rocblas@5.5.1 lapack-rocm: - spack_spec: rocsolver@5.5 + spack_spec: rocsolver@5.5.1 lapack: spack_spec: cray-libsci@23 mpi-rocm-gtl: - spack_spec: cray-mpich@8.1%cce +gtl + spack_spec: cray-mpich@8.1.26%cce@16.0.0 +gtl mpi-rocm-no-gtl: spack_spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl mpi-gcc: - spack_spec: cray-mpich@8.1%gcc ~gtl + spack_spec: cray-mpich@8.1.26%gcc@12.2.0 ~gtl fftw: spack_spec: intel-oneapi-mkl@2023.2.0 diff --git a/experiments/gromacs/openmp/ramble.yaml b/experiments/gromacs/openmp/ramble.yaml index 9c3945cba..daf99aead 100644 --- a/experiments/gromacs/openmp/ramble.yaml +++ b/experiments/gromacs/openmp/ramble.yaml @@ -48,7 +48,7 @@ ramble: spack_spec: fftw@3.3.10 +mpi+openmp compiler: default-compiler gromacs: - spack_spec: gromacs@main +mpi+openmp~hwloc + spack_spec: gromacs@main +mpi+openmp~hwloc~sycl~rocm~cuda compiler: default-compiler environments: gromacs: diff --git a/experiments/gromacs/rocm/ramble.yaml b/experiments/gromacs/rocm/ramble.yaml index 77406e606..cf0d3e597 100644 --- a/experiments/gromacs/rocm/ramble.yaml +++ b/experiments/gromacs/rocm/ramble.yaml @@ -34,28 +34,28 @@ ramble: pin: 'off' nb: '{target}' pme: 'auto' - bonded: '{target}' + bonded: 'cpu' update: '{target}' maxh: '0.05' nsteps: '1000' nstlist: '200' - npme: '0' + npme: '1' size: '1536' spack: packages: hipsycl: - spack_spec: hipsycl@23.10.0 + spack_spec: hipsycl@23.10.0+rocm amdgpu_target={rocm_arch} compiler: default-compiler gromacs: - spack_spec: gromacs@2023.3 +mpi+openmp+rocm~hwloc~double amdgpu_target={rocm_arch} + spack_spec: gromacs@2024 +mpi+openmp+sycl+rocm~hwloc~double amdgpu_target={rocm_arch} compiler: default-compiler environments: gromacs: packages: - - hipsycl + - default-mpi - blas-rocm - lapack-rocm - - mpi-rocm-gtl + - hipsycl - fftw - gromacs diff --git a/repo/gromacs/package.py b/repo/gromacs/package.py index 99f80170f..c9e7b25f2 100644 --- a/repo/gromacs/package.py +++ b/repo/gromacs/package.py @@ -8,9 +8,10 @@ import llnl.util.filesystem as fs from spack.package import * +from spack.pkg.benchpark.rocm_consistency import RocmConsistency as RocmConsistency -class Gromacs(CMakePackage, CudaPackage, ROCmPackage): +class Gromacs(CMakePackage, CudaPackage, ROCmPackage, RocmConsistency): """GROMACS is a molecular dynamics package primarily designed for simulations of proteins, lipids and nucleic acids. It was originally developed in the Biophysical Chemistry department of University of Groningen, and is now @@ -28,8 +29,12 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): git = "https://gitlab.com/gromacs/gromacs.git" maintainers("danielahlin", "eirrgang", "junghans") + license("BSD-2-Clause") + version("main", branch="main") version("master", branch="main", deprecated=True) + version("2024", sha256="04d226d52066a8bc3a42e00d6213de737b4ec292e26703065924ff01956801e2") + version("2023.4", sha256="e5d6c4d9e7ccacfaccb0888619bd21b5ea8911f82b410e68d6db5d40f695f231") version("2023.3", sha256="4ec8f8d0c7af76b13f8fd16db8e2c120e749de439ae9554d9f653f812d78d1cb") version("2023.2", sha256="bce1480727e4b2bb900413b75d99a3266f3507877da4f5b2d491df798f9fcdae") version("2023.1", sha256="eef2bb4a6cb6314cf9da47f26df2a0d27af4bf7b3099723d43601073ab0a42f4") @@ -96,6 +101,7 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): when="@2022: +cuda+mpi", description="Enable multi-GPU FFT support with cuFFTMp", ) + variant( "heffte", default=False, @@ -104,6 +110,7 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): ) variant("opencl", default=False, description="Enable OpenCL support") variant("sycl", default=False, when="@2021:", description="Enable SYCL support") + variant("sycl", default=True, when="@2021: +rocm", description="Enable SYCL support when using ROCm") variant( "intel-data-center-gpu-max", default=False, @@ -271,10 +278,14 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): depends_on("cmake@3.18.4:3", type="build", when="@main") depends_on("cmake@3.16.0:3", type="build", when="%fj") depends_on("cuda", when="+cuda") - depends_on("sycl", when="+sycl") + + with when("+rocm"): + depends_on("sycl") + depends_on("hip") + depends_on("rocfft") + depends_on("lapack") depends_on("blas") - depends_on("hipsycl", when="+rocm") depends_on("gcc", when="%oneapi ~intel_provided_gcc") depends_on("gcc", when="%intel ~intel_provided_gcc") @@ -286,8 +297,6 @@ class Gromacs(CMakePackage, CudaPackage, ROCmPackage): depends_on("nvhpc", when="+cufftmp") depends_on("heffte", when="+heffte") - conflicts("^hipsycl~rocm", when="+rocm") - requires( "%intel", "%oneapi", @@ -394,6 +403,10 @@ def patch(self): r"-gencode;arch=compute_20,code=sm_21;?", "", "cmake/gmxManageNvccConfig.cmake" ) + def setup_run_environment(self, env): + if self.compiler.extra_rpaths: + for rpath in self.compiler.extra_rpaths: + env.prepend_path("LD_LIBRARY_PATH", rpath) class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): @run_after("build") @@ -439,6 +452,22 @@ def cmake_args(self): # In other words, the mapping between package variants and the # GMX CMake variables is often non-trivial. + gmx_cc = spack_cc + gmx_cxx = spack_cxx + if "+rocm" in self.spec: + # The ROCm version requires the ROCm LLVM installation + gmx_cc = os.path.join(self.spec["llvm"].prefix.bin, "clang") + gmx_cxx = os.path.join(self.spec["llvm"].prefix.bin, "clang++") + if not fs.is_exe(gmx_cc) or not fs.is_exe(gmx_cxx): + gmx_cc = path.join(self.spec["llvm"].prefix.bin, "amdclang") + gmx_cxx = path.join(self.spec["llvm"].prefix.bin, "amdclang++") + if not fs.is_exe(gmx_cc) or not fs.is_exe(gmx_cxx): + raise InstallError( + "concretized LLVM dependency must provide a " + "valid clang/amdclang executable, found invalid: " + "{0}/{1}".format(gmx_cc, gmx_cxx) + ) + if "+mpi" in self.spec: options.append("-DGMX_MPI:BOOL=ON") if self.pkg.version < Version("2020"): @@ -462,8 +491,8 @@ def cmake_args(self): else: options.extend( [ - "-DCMAKE_C_COMPILER=%s" % spack_cc, - "-DCMAKE_CXX_COMPILER=%s" % spack_cxx, + "-DCMAKE_C_COMPILER=%s" % gmx_cc, + "-DCMAKE_CXX_COMPILER=%s" % gmx_cxx, "-DMPI_C_COMPILER=%s" % self.spec["mpi"].mpicc, "-DMPI_CXX_COMPILER=%s" % self.spec["mpi"].mpicxx, ] @@ -471,13 +500,17 @@ def cmake_args(self): else: options.extend( [ - "-DCMAKE_C_COMPILER=%s" % spack_cc, - "-DCMAKE_CXX_COMPILER=%s" % spack_cxx, + "-DCMAKE_C_COMPILER=%s" % gmx_cc, + "-DCMAKE_CXX_COMPILER=%s" % gmx_cxx, "-DGMX_MPI:BOOL=OFF", "-DGMX_THREAD_MPI:BOOL=ON", ] ) + # Here we cannot use spack_cc because we need also libstdc++ to be reachable + # Spack wrapper (spack_cc) hides includes/lib and CMake will fail + options.append("-DGMX_GPLUSPLUS_PATH=%s" % self.pkg.compiler.cxx) + if self.spec.satisfies("%aocc"): options.append("-DCMAKE_CXX_FLAGS=--stdlib=libc++") @@ -515,18 +548,14 @@ def cmake_args(self): options.append("-DGMX_GPU:STRING=CUDA") elif "+opencl" in self.spec: options.append("-DGMX_GPU:STRING=OpenCL") - elif "+rocm" in self.spec: - archs = self.spec.variants["amdgpu_target"].value - arch_str = ",".join(archs) - options.extend( - [ - "-DGMX_GPU:STRING=SYCL", - "-DGMX_SYCL_HIPSYCL=ON", - f"-DHIPSYCL_TARGETS=hip:{arch_str}", - ] - ) - elif "+sycl" in self.spec: + elif "+sycl" in self.spec or "+rocm" in self.spec: options.append("-DGMX_GPU:STRING=SYCL") + if "+rocm" in self.spec: + options.append("-DGMX_SYCL_HIPSYCL:BOOL=ON") + hipsycl_dir = os.path.join(self.spec["sycl"].prefix.lib, "cmake/hipSYCL/") + options.append(f"-Dhipsycl_DIR:STRING={hipsycl_dir}") + rocm_archs = ",".join(self.spec.variants["amdgpu_target"].value) + options.append(f"-DHIPSYCL_TARGETS:STRING=hip:{rocm_archs}") else: options.append("-DGMX_GPU:STRING=OFF") else: @@ -698,6 +727,9 @@ def cmake_args(self): "-DFFTWF_INCLUDE_DIR={0}".format(self.spec["acfl"].headers.directories[0]) ) options.append("-DFFTWF_LIBRARY={0}".format(self.spec["acfl"].libs.joined(";"))) + elif self.pkg.version >= Version("2023") and "+rocm" in self.spec: + # Use ROCm FFT library + options.append("-DGMX_GPU_FFT_LIBRARY=rocFFT") # Ensure that the GROMACS log files report how the code was patched # during the build, so that any problems are easier to diagnose. diff --git a/repo/hipsycl/package.py b/repo/hipsycl/package.py index 2de965139..430cbe465 100644 --- a/repo/hipsycl/package.py +++ b/repo/hipsycl/package.py @@ -11,51 +11,52 @@ from spack.package import * -class Hipsycl(CMakePackage, ROCmPackage): +class Hipsycl(CMakePackage, CudaPackage, ROCmPackage): """hipSYCL is an implementation of the SYCL standard programming model over NVIDIA CUDA/AMD HIP""" - homepage = "https://github.com/illuhad/hipSYCL" - url = "https://github.com/illuhad/hipSYCL/archive/v0.8.0.tar.gz" - git = "https://github.com/illuhad/hipSYCL.git" + homepage = "https://github.com/AdaptiveCpp/AdaptiveCpp" + url = "https://github.com/AdaptiveCpp/AdaptiveCpp/archive/v23.10.0.tar.gz" + git = "https://github.com/AdaptiveCpp/AdaptiveCpp.git" maintainers("nazavode") provides("sycl") + license("BSD-2-Clause") + version("stable", branch="stable", submodules=True) - version( - "23.10.0", - commit="3952b468c9da89edad9dff953cdcab0a3c3bf78c", - submodules=True, - get_full_repo=True, - ) - version("0.9.4", commit="99d9e24d462b35e815e0e59c1b611936c70464ae", submodules=True) - version("0.9.3", commit="51507bad524c33afe8b124804091b10fa25618dc", submodules=True) - version("0.9.2", commit="49fd02499841ae884c61c738610e58c27ab51fdb", submodules=True) - version("0.9.1", commit="fe8465cd5399a932f7221343c07c9942b0fe644c", submodules=True) - version("0.8.0", commit="2daf8407e49dd32ebd1c266e8e944e390d28b22a", submodules=True) + # New name ACPP + version("23.10.0", sha256="9ac3567c048a848f4e6eadb15b09750357ae399896e802b2f1dcaecf8a090064") + version("0.9.4", sha256="d9269c814f5e07b54a58bcef177950f222e22127c8399edc2e627d6b9e250763") + version("0.9.3", sha256="6a2e2d81bd21209ad0726d5aa377321e177fecb775ad93078259835be0931f51") + version("0.9.2", sha256="4b2308eb19b978a8528d55fe8c9fbb18d5be51aa0dd1a18a068946d8ddedebb1") + version("0.9.1", sha256="f4adbe283d21272d5b5a1431650971c29360453c167b8c3f6ed32611322aa4b1") + version("0.9.0", sha256="a7c565055fcb88dbca73693d3497fc8118e4f65b435e9bf136098a06b19dd8fc") + version("0.8.0", sha256="18e1a14d2f5f86a9bb4b799818c086d017a7e397a5f58bb92463a90355951f44") version("develop", branch="develop", submodules=True) - variant("cuda", default=False, description="Enable CUDA backend for SYCL kernels") + variant("opencl", default=False, description="Enable OpenCL backend for SYCL kernels") + variant("sscp", default=False, description="Enable SSCP compiler") + variant("intel", default=False, description="Enable Intel Level Zero backend for SYCL kernels") depends_on("cmake@3.5:", type="build") depends_on("boost +filesystem", when="@:0.8") depends_on("boost@1.67.0:1.69.0 +filesystem +fiber +context cxxstd=17", when="@0.9.1:") depends_on("python@3:") - depends_on("libllvm@8:", when="~cuda") - depends_on("libllvm@9:", when="+cuda") - depends_on("llvm+clang", when="^llvm") + depends_on("llvm@8: +clang", when="~cuda") + depends_on("llvm@9: +clang", when="+cuda") # hipSYCL 0.8.0 supported only LLVM 8-10: # (https://github.com/AdaptiveCpp/AdaptiveCpp/blob/v0.8.0/CMakeLists.txt#L29-L37) - depends_on("libllvm@8:10", when="@0.8.0") + depends_on("llvm@8:10", when="@0.8.0") # https://github.com/OpenSYCL/OpenSYCL/pull/918 was introduced after 0.9.4 - conflicts("^libllvm@16:", when="@:0.9.4") + conflicts("^llvm@16:", when="@:0.9.4") # LLVM PTX backend requires cuda7:10.1 (https://tinyurl.com/v82k5qq) depends_on("cuda@9:10.1", when="@0.8.1: +cuda ^llvm@9") depends_on("cuda@9:", when="@0.8.1: +cuda ^llvm@10:") # hipSYCL@:0.8.0 requires cuda@9:10.0 due to a known bug depends_on("cuda@9:10.0", when="@:0.8.0 +cuda") + depends_on("llvm@9: +clang", when="+rocm") conflicts( "%gcc@:4", @@ -81,16 +82,18 @@ def cmake_args(self): "-DWITH_CPU_BACKEND:Bool=TRUE", "-DWITH_ROCM_BACKEND:Bool={0}".format("TRUE" if "+rocm" in spec else "FALSE"), "-DWITH_CUDA_BACKEND:Bool={0}".format("TRUE" if "+cuda" in spec else "FALSE"), + "-DWITH_LEVEL_ZERO_BACKEND:Bool={0}".format("TRUE" if "+intel" in spec else "FALSE"), + "-DWITH_OPENCL_BACKEND:Bool={0}".format("TRUE" if "+opencl" in spec else "FALSE"), + "-DWITH_SSCP_COMPILER:Bool={0}".format("TRUE" if "+sscp" in spec else "FALSE"), # prevent hipSYCL's cmake to look for other LLVM installations # if the specified one isn't compatible "-DDISABLE_LLVM_VERSION_CHECK:Bool=TRUE", ] - args += [f"-DACPP_VERSION_SUFFIX=spack-{self.version}"] - if "+rocm" in spec: - args += [f"-DHIP_CXX_COMPILER={self.compiler.cxx}"] + if self.version >= Version("23.10.0"): + args.append("-DACPP_VERSION_SUFFIX={0}".format(self.version)) # LLVM directory containing all installed CMake files # (e.g.: configs consumed by client projects) - llvm_cmake_dirs = filesystem.find(spec["libllvm"].prefix, "LLVMExports.cmake") + llvm_cmake_dirs = filesystem.find(spec["llvm"].prefix.lib, "LLVMExports.cmake") if len(llvm_cmake_dirs) != 1: raise InstallError( "concretized llvm dependency must provide " @@ -98,9 +101,10 @@ def cmake_args(self): "files, found: {0}".format(llvm_cmake_dirs) ) args.append("-DLLVM_DIR:String={0}".format(path.dirname(llvm_cmake_dirs[0]))) + # clang internal headers directory llvm_clang_include_dirs = filesystem.find( - spec["libllvm"].prefix, "__clang_cuda_runtime_wrapper.h" + spec["llvm"].prefix.lib, "__clang_cuda_runtime_wrapper.h" ) if len(llvm_clang_include_dirs) != 1: raise InstallError( @@ -111,18 +115,32 @@ def cmake_args(self): args.append( "-DCLANG_INCLUDE_PATH:String={0}".format(path.dirname(llvm_clang_include_dirs[0])) ) - # target clang++ executable - llvm_clang_bin = path.join(spec["libllvm"].prefix.bin, "clang++") + # Find the right LLVM compiler + llvm_clang_bin = path.join(spec["llvm"].prefix.bin, "clang") + llvm_clang_bin_cpp = path.join(spec["llvm"].prefix.bin, "clang++") if not filesystem.is_exe(llvm_clang_bin): - raise InstallError( - "concretized llvm dependency must provide a " - "valid clang++ executable, found invalid: " - "{0}".format(llvm_clang_bin) - ) + llvm_clang_bin = path.join(spec["llvm"].prefix.bin, "amdclang") + llvm_clang_bin_cpp = path.join(spec["llvm"].prefix.bin, "amdclang++") + if not filesystem.is_exe(llvm_clang_bin): + raise InstallError( + "concretized LLVM dependency must provide a " + "valid clang/amdclang executable, found invalid: " + "{0}".format(llvm_clang_bin) + ) args.append("-DCLANG_EXECUTABLE_PATH:String={0}".format(llvm_clang_bin)) + args.append("-DCMAKE_C_COMPILER:String={0}".format(llvm_clang_bin)) + args.append("-DCMAKE_CXX_COMPILER:String={0}".format(llvm_clang_bin_cpp)) + # explicit CUDA toolkit if "+cuda" in spec: args.append("-DCUDA_TOOLKIT_ROOT_DIR:String={0}".format(spec["cuda"].prefix)) + if "+rocm" in spec: + # FIXME: here spec["rocm"].prefix does not work + # Instead (temporary solution: we use HIP prefix and + # remove the "hip/" part of the path which is the ROCm path + rocm_path = path.split(spec["hip"].prefix[:-1])[0] + args.append("-DROCM_PATH:String={0}".format(rocm_path)) + return args @run_after("install") @@ -138,34 +156,32 @@ def filter_config_file(self): config = json.load(f) # 1. Fix compiler: use the real one in place of the Spack wrapper config["default-cpu-cxx"] = self.compiler.cxx + # 2. Fix stdlib: we need to make sure cuda-enabled binaries find + # the libc++.so and libc++abi.so dyn linked to the sycl + # ptx backend if "+cuda" in self.spec: - # 2. Fix stdlib: we need to make sure cuda-enabled binaries find - # the libc++.so and libc++abi.so dyn linked to the sycl - # ptx backend rpaths = set() so_paths = filesystem.find_libraries( - "libc++", self.spec["libllvm"].prefix, shared=True, recursive=True + "libc++", self.spec["llvm"].prefix, shared=True, recursive=True ) if len(so_paths) != 1: raise InstallError( "concretized llvm dependency must provide a " "unique directory containing libc++.so, " - "found: {0}".format(so_paths) + "found: \"{0}\"".format(so_paths) ) rpaths.add(path.dirname(so_paths[0])) so_paths = filesystem.find_libraries( - "libc++abi", self.spec["libllvm"].prefix, shared=True, recursive=True + "libc++abi", self.spec["llvm"].prefix, shared=True, recursive=True ) if len(so_paths) != 1: raise InstallError( "concretized llvm dependency must provide a " "unique directory containing libc++abi, " - "found: {0}".format(so_paths) + "found: \"{0}\"".format(so_paths) ) rpaths.add(path.dirname(so_paths[0])) - config["default-cuda-link-line"] += " " + " ".join( - "-rpath {0}".format(p) for p in rpaths - ) + config["default-cuda-link-line"] += " " + " ".join("-rpath {0}".format(p) for p in rpaths) # Replace the installed config file with open(config_file_path, "w") as f: json.dump(config, f, indent=2) diff --git a/repo/rocm-consistency/package.py b/repo/rocm-consistency/package.py new file mode 100644 index 000000000..94df70f59 --- /dev/null +++ b/repo/rocm-consistency/package.py @@ -0,0 +1,41 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * +from spack.package_base import PackageBase + + +class RocmConsistency(PackageBase): + with when("+rocm"): + for ver in [ + "5.1.0", + "5.1.3", + "5.2.0", + "5.2.1", + "5.2.3", + "5.3.0", + "5.3.3", + "5.4.0", + "5.4.3", + "5.5.0", + "5.5.1", + "5.6.0", + "5.6.1", + "5.7.0", + "5.7.1", + "6.0.0", + "6.0.2", + ]: + depends_on(f"hip@{ver}", when=f"%rocmcc@{ver} ^hip") + depends_on(f"hsakmt-roct@{ver}", when=f"^hip@{ver}") + depends_on(f"hsa-rocr-dev@{ver}", when=f"^hip@{ver}") + depends_on(f"comgr@{ver}", when=f"^hip@{ver}") + if ver >= "5.5.1": + depends_on(f"llvm@16.0.0-{ver} +clang~lld~lldb", when=f"^hip@{ver}") + else: + depends_on(f"llvm@15.0.0-{ver} +clang~lld~lldb", when=f"^hip@{ver}") + depends_on(f"llvm-amdgpu@{ver} +rocm-device-libs", when=f"^hip@{ver}") + depends_on(f"rocminfo@{ver}", when=f"^hip@{ver}") + depends_on(f"roctracer-dev-api@{ver}", when=f"^hip@{ver}")