diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/openmp/execute_experiment.tpl +++ b/experiments/kripke/openmp/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml index 0ede5c91f..670af7156 100644 --- a/experiments/kripke/openmp/ramble.yaml +++ b/experiments/kripke/openmp/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,41 +18,40 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: set: OMP_NUM_THREADS: '{omp_num_threads}' variables: n_ranks: '{processes_per_node} * {n_nodes}' - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' experiment_setup: '' - processes_per_node: ['8', '4'] - n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks - omp_num_threads: '{threads_per_node_core} * {n_nodes}' + processes_per_node: ['1', '16'] + n_nodes: '1' + omp_num_threads: ['32', '2'] + arch: 'OpenMP' experiments: - kripke_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_threads: - - n # TODO: Filter matrix - - threads_per_node_core # TODO: Filter matrix + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '32' + nzz: '32' + npx: ['1', '4'] + npy: ['1', '2'] + npz: ['1', '2'] spack: concretized: true packages: - kripke-omp: - spack_spec: kripke@develop +mpi+openmp + kripke: + spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-omp: + kripke: packages: - default-mpi - - kripke-omp + - kripke + - '{modifier_package_name}' diff --git a/repo/blt/package.py b/repo/blt/package.py new file mode 100644 index 000000000..64725f485 --- /dev/null +++ b/repo/blt/package.py @@ -0,0 +1,11 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * +from spack.pkg.builtin.blt import Blt as BuiltinBlt + + +class Blt(BuiltinBlt): + version("0.6.2", sha256="84b663162957c1fe0e896ac8e94cbf2b6def4a152ccfa12a293db14fb25191c8") diff --git a/repo/camp/libstdc++-13-missing-header.patch b/repo/camp/libstdc++-13-missing-header.patch new file mode 100644 index 000000000..d3c642abe --- /dev/null +++ b/repo/camp/libstdc++-13-missing-header.patch @@ -0,0 +1,24 @@ +diff --git a/include/camp/resource.hpp b/include/camp/resource.hpp +index 58df206..763aa20 100644 +--- a/include/camp/resource.hpp ++++ b/include/camp/resource.hpp +@@ -12,6 +12,7 @@ http://github.com/llnl/camp + #define __CAMP_RESOURCE_HPP + + #include ++#include + #include + #include + #include +diff --git a/include/camp/resource/host.hpp b/include/camp/resource/host.hpp +index c765175..4428c57 100644 +--- a/include/camp/resource/host.hpp ++++ b/include/camp/resource/host.hpp +@@ -13,6 +13,7 @@ http://github.com/llnl/camp + + #include "camp/resource/event.hpp" + #include "camp/resource/platform.hpp" ++#include + + namespace camp + { diff --git a/repo/camp/package.py b/repo/camp/package.py new file mode 100644 index 000000000..241648029 --- /dev/null +++ b/repo/camp/package.py @@ -0,0 +1,108 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import glob + +from spack.package import * + + +def hip_repair_options(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + "-DHIP_CLANG_INCLUDE_PATH=" + + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0] + ) + + +def hip_repair_cache(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + cmake_cache_path( + "HIP_CLANG_INCLUDE_PATH", + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0], + ) + ) + + +class Camp(CMakePackage, CudaPackage, ROCmPackage): + """ + Compiler agnostic metaprogramming library providing concepts, + type operations and tuples for C++ and cuda + """ + + homepage = "https://github.com/LLNL/camp" + git = "https://github.com/LLNL/camp.git" + url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" + + maintainers("trws") + + license("BSD-3-Clause") + + version("main", branch="main", submodules="True") + version( + "2024.02.1", + tag="v2024.02.1", + commit="79c320fa09db987923b56884afdc9f82f4b70fc4", + submodules=False, + ) + version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02") + version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") + version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") + version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") + version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") + version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") + version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") + + # TODO: figure out gtest dependency and then set this default True. + variant("tests", default=False, description="Build tests") + variant("openmp", default=False, description="Build OpenMP support") + + depends_on("cub", when="+cuda") + + depends_on("blt", type="build") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2024.01") + depends_on("blt@0.6.2", type="build", when="@2024.02:") + + patch("libstdc++-13-missing-header.patch", when="@:2022.10") + + conflicts("^blt@:0.3.6", when="+rocm") + + def cmake_args(self): + spec = self.spec + + options = [] + + options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) + + options.append("-DENABLE_OPENMP=" + ("On" if "+openmp" in spec else "Off")) + if "+cuda" in spec: + options.extend( + ["-DENABLE_CUDA=ON", "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix)] + ) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) + options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) + flag = "-arch sm_{0}".format(cuda_arch[0]) + options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) + else: + options.append("-DENABLE_CUDA=OFF") + + if "+rocm" in spec: + options.extend(["-DENABLE_HIP=ON", "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)]) + + hip_repair_options(options, spec) + + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str)) + else: + options.append("-DENABLE_HIP=OFF") + + options.append(self.define_from_variant("ENABLE_TESTS", "tests")) + + return options diff --git a/repo/chai/package.py b/repo/chai/package.py new file mode 100644 index 000000000..152d94dcc --- /dev/null +++ b/repo/chai/package.py @@ -0,0 +1,228 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import socket + +from spack.package import * + + +class Chai(CachedCMakePackage, CudaPackage, ROCmPackage): + """ + Copy-hiding array interface for data migration between memory spaces + """ + + homepage = "https://github.com/LLNL/CHAI" + git = "https://github.com/LLNL/CHAI.git" + tags = ["ecp", "e4s", "radiuss"] + + maintainers("davidbeckingsale") + + license("BSD-3-Clause") + + version("develop", branch="develop", submodules=False) + version("main", branch="main", submodules=False) + version( + "2024.02.0", + tag="v2024.02.0", + commit="31773a2f0d30f3f64c82939f60fc4da32cf33261", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="f0b809de1ac194376866b3ac0f5933d4146ec09e", + submodules=False, + ) + version( + "2.4.0", tag="v2.4.0", commit="77d22da28187245a2c5454cf471c0c352bd98ad7", submodules=True + ) + version( + "2.3.0", tag="v2.3.0", commit="42f3fbcc0b966227b40b4467dc919a4c24f07196", submodules=True + ) + version( + "2.2.2", tag="v2.2.2", commit="56e75fc0f805b2746f3992af0c00c474513e3b24", submodules=True + ) + version( + "2.2.1", tag="v2.2.1", commit="c912f583828ea5963850816e3e232cc45608ccf7", submodules=True + ) + version( + "2.2.0", tag="v2.2.0", commit="18536c61a4817db6b3b3025f35e2dd3ae532330c", submodules=True + ) + version( + "2.1.1", tag="v2.1.1", commit="496911e00d15c350560860f7964cd5fb5ab7f515", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="fff02768068a64970b34760a1041585319edee87", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="63139cf45443b1266950826b165e042c7679b557", submodules=True + ) + version( + "1.2.0", tag="v1.2.0", commit="7bb5bc12e4508db45910d8e2b98444687da7ebf6", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True + ) + version("1.0", tag="v1.0", commit="501a098ad879dc8deb4a74fcfe8c08c283a10627", submodules=True) + + variant("enable_pick", default=False, description="Enable pick method") + variant( + "separable_compilation", + default=True, + description="Build with CUDA_SEPARABLE_COMPILATION flag on ", + ) + variant("shared", default=True, description="Build Shared Libs") + variant("raja", default=False, description="Build plugin for RAJA") + variant("benchmarks", default=False, description="Build benchmarks.") + variant("examples", default=True, description="Build examples.") + variant("openmp", default=False, description="Build using OpenMP") + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant("tests", default=False, description="Build tests") + + depends_on("cmake@3.8:", type="build") + depends_on("cmake@3.9:", type="build", when="+cuda") + depends_on("cmake@3.14:", when="@2022.03.0:") + + depends_on("blt@0.6.2", type="build", when="@2024.02:") + depends_on("blt@0.5.0:", type="build", when="@2022.03.0:2024.01") + depends_on("blt@0.4.1:", type="build", when="@2.4.0:") + depends_on("blt@0.4.0:", type="build", when="@2.3.0") + depends_on("blt@0.3.6:", type="build", when="@:2.2.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("umpire") + depends_on("umpire@2024.02:", when="@2024.02:") + depends_on("umpire@2022.03.0:", when="@2022.03.0:2024.01") + depends_on("umpire@6.0.0", when="@2.4.0") + depends_on("umpire@4.1.2", when="@2.2.0:2.3.0") + depends_on("umpire@main", when="@main") + + with when("+cuda"): + depends_on("umpire+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("umpire+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm"): + depends_on("umpire+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "umpire+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) + ) + + with when("+raja"): + depends_on("raja~openmp", when="~openmp") + depends_on("raja+openmp", when="+openmp") + depends_on("raja@0.14.0", when="@2.4.0") + depends_on("raja@0.13.0", when="@2.3.0") + depends_on("raja@0.12.0", when="@2.2.0:2.2.2") + depends_on("raja@2022.03.0:", when="@2022.03.0:") + depends_on("raja@main", when="@main") + + with when("+cuda"): + depends_on("raja+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("raja+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + with when("+rocm"): + depends_on("raja+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "raja+rocm amdgpu_target={0}".format(arch), + when="amdgpu_target={0}".format(arch), + ) + + conflicts("+benchmarks", when="~tests") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + ) + + def initconfig_compiler_entries(self): + spec = self.spec + entries = super().initconfig_compiler_entries() + if "+rocm" in spec: + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) + + if "+cuda" in spec: + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + if "+separable_compilation" in spec: + entries.append(cmake_cache_option("CMAKE_CUDA_SEPARABLE_COMPILATION", True)) + entries.append(cmake_cache_option("CUDA_SEPARABLE_COMPILATION", True)) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) + entries.append( + cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) + ) + flag = "-arch sm_{0}".format(cuda_arch[0]) + entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if "+rocm" in spec: + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append( + cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) + ) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "CHAI_" if spec.satisfies("@2022.03.0:") else "" + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if "+raja" in spec: + entries.append(cmake_cache_option("{}ENABLE_RAJA_PLUGIN".format(option_prefix), True)) + entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) + entries.append( + cmake_cache_option("{}ENABLE_PICK".format(option_prefix), "+enable_pick" in spec) + ) + if spec.satisfies("@2024.02:"): + entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.lib64.cmake.umpire)) + else: + entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.share.umpire.cmake)) + entries.append(cmake_cache_option("ENABLE_TESTS", "+tests" in spec)) + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "+benchmarks" in spec)) + entries.append( + cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) + ) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) + + return entries + + def cmake_args(self): + options = [] + return options diff --git a/repo/kripke/application.py b/repo/kripke/application.py index e9d0ba2a3..61a79f7f6 100644 --- a/repo/kripke/application.py +++ b/repo/kripke/application.py @@ -17,44 +17,102 @@ class Kripke(SpackApplication): 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', 'mpi','network-latency-bound','network-collectives'] - executable('p1', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 1' + - ' -keepT', use_mpi=True) - - executable('p2', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 2' + - ' -keepT', use_mpi=True) - - workload('problem1', executables=['p1']) - workload('problem2', executables=['p2']) - - workload_variable('px', default='2', - description='px', - workloads=['problem1', 'problem2']) - workload_variable('py', default='2', - description='py', - workloads=['problem1', 'problem2']) - workload_variable('pz', default='2', - description='pz', - workloads=['problem1', 'problem2']) - workload_variable('nx', default='220', - description='nx', - workloads=['problem1', 'problem2']) - workload_variable('ny', default='220', - description='ny', - workloads=['problem1', 'problem2']) - workload_variable('nz', default='220', - description='nz', - workloads=['problem1', 'problem2']) - - figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + executable('kripke', 'kripke.exe' + + ' --groups {ngroups}' + + ' --legendre {lorder}' + + ' --quad {nquad}' + + ' --zones {nzx},{nzy},{nzz}' + + ' --sigt {sigt0},{sigt1},{sigt2}' + + ' --sigs {sigs0},{sigs1},{sigs2}' + + ' --arch {arch}' + + ' --layout {layout}' + +# ' --pdist {lout}' + + ' --procs {npx},{npy},{npz}' + + ' --dset {ds}' + + ' --gset {gs}' + + ' --zset {nzsetx},{nzsety},{nzsetz}' + + ' --niter {niter}' + + ' --pmethod {method}', use_mpi=True) - #TODO: Fix the FOM success_criteria(...) - success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') + workload('kripke', executables=['kripke']) + + workload_variable('ngroups', default='32', + description='Number of energy groups. (Default: --groups 32)', + workloads=['kripke']) + workload_variable('lorder', default='4', + description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', + workloads=['kripke']) + workload_variable('nquad', default='96', + description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in x. (Default: 16)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in y. (Default: 16)', + workloads=['kripke']) + workload_variable('nzz', default='16', + description='Number of zones in z. (Default: 16)', + workloads=['kripke']) + workload_variable('sigt0', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt1', default='0.0001', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt2', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs0', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs1', default='0.00005', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs2', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('arch', default='OpenMP', + description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', + workloads=['kripke']) + workload_variable('layout', default='DGZ', + description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', + workloads=['kripke']) +# workload_variable('lout', default='0', +# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', +# workloads=['kripke']) + workload_variable('npx', default='1', + description='Number of MPI ranks in x dimension', + workloads=['kripke']) + workload_variable('npy', default='1', + description='Number of MPI ranks in y dimension', + workloads=['kripke']) + workload_variable('npz', default='1', + description='Number of MPI ranks in z dimension', + workloads=['kripke']) + workload_variable('ds', default='8', + description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', + workloads=['kripke']) + workload_variable('gs', default='1', + description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', + workloads=['kripke']) + workload_variable('nzsetx', default='1', + description='Number of zone-sets in x', + workloads=['kripke']) + workload_variable('nzsety', default='1', + description='Number of zone-sets in y', + workloads=['kripke']) + workload_variable('nzsetz', default='1', + description='Number of zone-sets in z', + workloads=['kripke']) + workload_variable('niter', default='10', + description='Number of solver iterations to run. (Default: --niter 10)', + workloads=['kripke']) + workload_variable('method', default='sweep', + description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', + workloads=['kripke']) - def evaluate_success(self): - return True + #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + + #TODO: Fix the FOM success_criteria(...) + #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 6bf335f35..7aed1369e 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -56,17 +56,32 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") + depends_on("blt@0.6.2", type="build") + conflicts("^blt@:0.3.6", when="+rocm") + aligned_versions = ["2024.02"] - constrain_chai = list(f"^chai@{v}" for v in aligned_versions) - requires(*aligned_versions, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") - + #constrain_chai = list(f"^chai@{v}" for v in aligned_versions) + #requires(*constrain_chai, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") + for v in aligned_versions: depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") depends_on(f"umpire@{v}~examples", when=f"^chai@{v}") depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}") - - depends_on("blt@0.6.2", type="build") - conflicts("^blt@:0.3.6", when="+rocm") + depends_on(f"camp@{v}", when=f"^chai@{v}") + + depends_on("chai+openmp", when="+openmp") + depends_on("chai~openmp", when="~openmp") + depends_on("chai+cuda", when="+cuda") + depends_on("chai~cuda", when="~cuda") + depends_on("chai+rocm", when="+rocm") + depends_on("chai~rocm", when="~rocm") + + depends_on("umpire+openmp", when="+openmp") + depends_on("umpire~openmp", when="~openmp") + depends_on("umpire+cuda", when="+cuda") + depends_on("umpire~cuda", when="~cuda") + depends_on("umpire+rocm", when="+rocm") + depends_on("umpire~rocm", when="~rocm") def cmake_args(self): spec = self.spec @@ -83,6 +98,9 @@ def cmake_args(self): ] ) + if "+openmp" in spec: + args.append("-DENABLE_OPENMP=ON") + if "+caliper" in spec: args.append("-DENABLE_CALIPER=ON") diff --git a/repo/raja/package.py b/repo/raja/package.py new file mode 100644 index 000000000..41e35ac95 --- /dev/null +++ b/repo/raja/package.py @@ -0,0 +1,17 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * +from spack.pkg.builtin.raja import Raja as BuiltinRaja + + +class Raja(BuiltinRaja): + version( + "2024.02.1", + tag="v2024.02.1", + commit="3ada0950b0774ec907d30a9eceaf6af7478b833b", + submodules=False, + ) + depends_on("blt@0.6.2", type="build", when="@2024.02:") diff --git a/repo/umpire/camp_target_umpire_3.0.0.patch b/repo/umpire/camp_target_umpire_3.0.0.patch new file mode 100644 index 000000000..2deb565cd --- /dev/null +++ b/repo/umpire/camp_target_umpire_3.0.0.patch @@ -0,0 +1,18 @@ +diff --git a/umpire-config.cmake.in b/umpire-config.cmake.in +index a98ad5fe..4e54e173 100644 +--- a/umpire-config.cmake.in ++++ b/umpire-config.cmake.in +@@ -7,6 +7,13 @@ + get_filename_component(UMPIRE_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) + set(UMPIRE_INCLUDE_DIRS "@CMAKE_INSTALL_PREFIX@/include") + ++if (NOT TARGET camp) ++ if (NOT DEFINED camp_DIR) ++ set(camp_DIR @CMAKE_INSTALL_PREFIX@/lib/cmake/camp) ++ endif () ++ find_package(camp REQUIRED) ++endif () ++ + set(Umpire_VERSION_MAJOR @Umpire_VERSION_MAJOR@) + set(Umpire_VERSION_MINOR @Umpire_VERSION_MINOR@) + set(Umpire_VERSION_PATCH @Umpire_VERSION_PATCH@) diff --git a/repo/umpire/cmake_version_check.patch b/repo/umpire/cmake_version_check.patch new file mode 100644 index 000000000..05f80e0b6 --- /dev/null +++ b/repo/umpire/cmake_version_check.patch @@ -0,0 +1,13 @@ +diff --git a/src/umpire/CMakeLists.txt b/src/umpire/CMakeLists.txt +index 18518f13..c2e45e03 100644 +--- a/src/umpire/CMakeLists.txt ++++ b/src/umpire/CMakeLists.txt +@@ -105,7 +105,7 @@ blt_add_library( + DEPENDS_ON ${umpire_depends} + DEFINES ${umpire_defines}) + +-if (NOT WIN32 AND CMAKE_VERSION VERSION_GREATER 3.17) ++if (NOT WIN32 AND NOT ${CMAKE_VERSION} VERSION_LESS 3.18) + target_link_options( + umpire INTERFACE + $>:-rdynamic>> diff --git a/repo/umpire/missing_header_for_numeric_limits.patch b/repo/umpire/missing_header_for_numeric_limits.patch new file mode 100644 index 000000000..6e89e3707 --- /dev/null +++ b/repo/umpire/missing_header_for_numeric_limits.patch @@ -0,0 +1,12 @@ +diff --git a/src/umpire/util/allocation_statistics.cpp b/src/umpire/util/allocation_statistics.cpp +index 597ae61f..0c3f7865 100644 +--- a/src/umpire/util/allocation_statistics.cpp ++++ b/src/umpire/util/allocation_statistics.cpp +@@ -8,6 +8,7 @@ + #include "umpire/util/allocation_statistics.hpp" + + #include ++#include + + namespace umpire { + namespace util { diff --git a/repo/umpire/package.py b/repo/umpire/package.py new file mode 100644 index 000000000..7c23151c4 --- /dev/null +++ b/repo/umpire/package.py @@ -0,0 +1,376 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import socket + +import llnl.util.tty as tty + +from spack.package import * +from spack.pkg.builtin.camp import hip_repair_cache + + +class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): + """An application-focused API for memory management on NUMA & GPU + architectures""" + + homepage = "https://github.com/LLNL/Umpire" + git = "https://github.com/LLNL/Umpire.git" + tags = ["radiuss", "e4s"] + + maintainers("davidbeckingsale") + + license("MIT") + + version("develop", branch="develop", submodules=False) + version("main", branch="main", submodules=False) + version( + "2024.02.1", + tag="v2024.02.1", + commit="3058d562fc707650e904f9321b1ee9bcebad3ae2", + submodules=True, + ) + version( + "2022.10.0", + tag="v2022.10.0", + commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a", + submodules=False, + ) + version( + "2022.03.1", + tag="v2022.03.1", + commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b", + submodules=False, + ) + version( + "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True + ) + version( + "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True + ) + version( + "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True + ) + version( + "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True + ) + version( + "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True + ) + version( + "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True + ) + version( + "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True + ) + version( + "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True + ) + version( + "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True + ) + version( + "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True + ) + version( + "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True + ) + version( + "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True + ) + version( + "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True + ) + version( + "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True + ) + version( + "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True + ) + version( + "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True + ) + version( + "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True + ) + version( + "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True + ) + version( + "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True + ) + version( + "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True + ) + version( + "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True + ) + version( + "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True + ) + version( + "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True + ) + version( + "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True + ) + + patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:") + patch("camp_target_umpire_3.0.0.patch", when="@3.0.0") + patch("cmake_version_check.patch", when="@4.1") + patch("missing_header_for_numeric_limits.patch", when="@4.1:5.0.1") + + # export targets when building pre-6.0.0 release with BLT 0.4.0+ + patch( + "https://github.com/LLNL/Umpire/commit/5773ce9af88952c8d23f9bcdcb2e503ceda40763.patch?full_index=1", + sha256="f3b21335ce5cf9c0fecc852a94dfec90fb5703032ac97f9fee104af9408d8899", + when="@:5.0.1 ^blt@0.4:", + ) + + # https://github.com/LLNL/Umpire/pull/805 + patch( + "https://github.com/LLNL/Umpire/pull/805/commits/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1", + sha256="7ed5d2c315a3b31e339f664f6108e32d7cb4cb8e9f22e5c78a65ba02625ccc09", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/816 + patch( + "https://github.com/LLNL/Umpire/pull/816/commits/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1", + sha256="0f43cad7cdaec3c225ab6414ab9f81bd405a1157abf5a508e515bcb6ca53326d", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/853 + patch( + "https://github.com/LLNL/Umpire/commit/4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256.patch?full_index=1", + sha256="c9ddae1f4212cef72e1050b6ac482ce5b795dad4977d2462cff2e884b8d7aff5", + when="@2022.10:2023.06", + ) + + variant("fortran", default=False, description="Build C/Fortran API") + variant("c", default=True, description="Build C API") + variant("numa", default=False, description="Enable NUMA support") + variant("shared", default=True, description="Enable Shared libs") + variant("openmp", default=False, description="Build with OpenMP support") + variant("deviceconst", default=False, description="Enables support for constant device memory") + variant("examples", default=True, description="Build Umpire Examples") + variant( + "tests", + default="none", + values=("none", "basic", "benchmarks"), + multi=False, + description="Tests to run", + ) + variant("device_alloc", default=True, description="Build Umpire Device Allocator") + + depends_on("cmake@3.8:", type="build") + depends_on("cmake@3.9:", when="+cuda", type="build") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") + + depends_on("blt", type="build") + depends_on("blt@0.6.2", type="build", when="@2024.02:") + depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2022.03.1") + depends_on("blt@0.4.1", type="build", when="@6.0.0") + depends_on("blt@0.4.0:", type="build", when="@4.1.3:5.0.1") + depends_on("blt@0.3.6:", type="build", when="@:4.1.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("camp", when="@5.0.0:") + depends_on("camp@0.2.2:0.2.3", when="@6.0.0") + depends_on("camp@0.1.0", when="@5.0.0:5.0.1") + depends_on("camp@2022.03.2:", when="@2022.03.0:2024.01") + depends_on("camp@2024.02:", when="@2024.02:") + depends_on("camp@main", when="@main") + depends_on("camp@main", when="@develop") + depends_on("camp+openmp", when="+openmp") + depends_on("camp~cuda", when="~cuda") + depends_on("camp~rocm", when="~rocm") + + with when("@5.0.0:"): + with when("+cuda"): + depends_on("camp+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("camp+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm"): + depends_on("camp+rocm") + for arch_ in ROCmPackage.amdgpu_targets: + depends_on( + "camp+rocm amdgpu_target={0}".format(arch_), + when="amdgpu_target={0}".format(arch_), + ) + + conflicts("+numa", when="@:0.3.2") + conflicts("~c", when="+fortran", msg="Fortran API requires C API") + conflicts("+device_alloc", when="@:2022.03.0") + + # device allocator exports device code, which requires static libs + # currently only available for cuda. + conflicts("+shared", when="+cuda") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + ) + + def initconfig_compiler_entries(self): + spec = self.spec + entries = super().initconfig_compiler_entries() + + if "+rocm" in spec: + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if "+fortran" in spec and self.compiler.fc is not None: + entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) + else: + entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) + + entries.append(cmake_cache_option("{}ENABLE_C".format(option_prefix), "+c" in spec)) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if "+cuda" in spec: + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) + entries.append( + cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) + ) + flag = "-arch sm_{0}".format(cuda_arch[0]) + entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) + + entries.append( + cmake_cache_option( + "{}ENABLE_DEVICE_CONST".format(option_prefix), spec.satisfies("+deviceconst") + ) + ) + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if "+rocm" in spec: + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) + hip_repair_cache(entries, spec) + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append( + cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) + ) + entries.append( + cmake_cache_string("CMAKE_HIP_ARCHITECTURES", "{0}".format(arch_str)) + ) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if spec.satisfies("@5.0.0:"): + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + entries.append(cmake_cache_option("{}ENABLE_NUMA".format(option_prefix), "+numa" in spec)) + entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "tests=benchmarks" in spec)) + entries.append( + cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) + ) + entries.append(cmake_cache_option("{}ENABLE_DOCS".format(option_prefix), False)) + entries.append( + cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", "+device_alloc" in spec) + ) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) + entries.append(cmake_cache_option("ENABLE_TESTS", "tests=none" not in spec)) + + return entries + + def cmake_args(self): + options = [] + return options + + def test(self): + """Perform stand-alone checks on the installed package.""" + if self.spec.satisfies("@:1") or not os.path.isdir(self.prefix.bin): + tty.info("Skipping: checks not installed in bin for v{0}".format(self.version)) + return + + # Run a subset of examples PROVIDED installed + # tutorials with readily checkable outputs. + checks = { + "malloc": ["99 should be 99"], + "recipe_dynamic_pool_heuristic": ["in the pool", "releas"], + "recipe_no_introspection": ["has allocated", "used"], + "strategy_example": ["Available allocators", "HOST"], + "tut_copy": ["Copied source data"], + "tut_introspection": ["Allocator used is HOST", "size of the allocation"], + "tut_memset": ["Set data from HOST"], + "tut_move": ["Moved source data", "HOST"], + "tut_reallocate": ["Reallocated data"], + "vector_allocator": [""], + } + + for exe in checks: + expected = checks[exe] + reason = "test: checking output from {0}".format(exe) + self.run_test( + exe, + [], + expected, + 0, + installed=False, + purpose=reason, + skip_missing=True, + work_dir=self.prefix.bin, + ) diff --git a/repo/umpire/std-filesystem-pr784.patch b/repo/umpire/std-filesystem-pr784.patch new file mode 100644 index 000000000..677144214 --- /dev/null +++ b/repo/umpire/std-filesystem-pr784.patch @@ -0,0 +1,33 @@ +diff -ruN spack-src/cmake/SetupCompilerFlags.cmake spack-src-patched/cmake/SetupCompilerFlags.cmake +--- spack-src/cmake/SetupCompilerFlags.cmake 2022-10-12 08:05:03.538390165 -0700 ++++ spack-src-patched/cmake/SetupCompilerFlags.cmake 2022-10-12 09:47:56.317645003 -0700 +@@ -11,20 +11,20 @@ + + message(STATUS "Checking for std::filesystem") + +-include(CheckCXXSourceCompiles) +-check_cxx_source_compiles( +- "#include +- #include +- +- int main(int, char**) +- { ++blt_check_code_compiles(CODE_COMPILES UMPIRE_ENABLE_FILESYSTEM ++ SOURCE_STRING ++[=[ ++#include ++#include + ++int main(int, char**) ++{ + auto path = std::filesystem::path(\".\"); + (void)(path); + + return 0; +- }" +- UMPIRE_ENABLE_FILESYSTEM) ++} ++]=]) + + if (UMPIRE_ENABLE_FILESYSTEM) + message(STATUS "std::filesystem found")