From 621d6fea2ce987ec6979a8b3f9181d9e3a76758f Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Sun, 28 Apr 2024 11:32:03 -0700 Subject: [PATCH 01/14] OpenMP implementation of Kripke --- .../kripke/openmp/execute_experiment.tpl | 5 - experiments/kripke/openmp/ramble.yaml | 46 +-- repo/blt/package.py | 11 + repo/camp/libstdc++-13-missing-header.patch | 24 ++ repo/camp/package.py | 108 +++++ repo/chai/package.py | 228 +++++++++++ repo/kripke/application.py | 136 +++++-- repo/kripke/package.py | 30 +- repo/raja/package.py | 17 + repo/umpire/camp_target_umpire_3.0.0.patch | 18 + repo/umpire/cmake_version_check.patch | 13 + .../missing_header_for_numeric_limits.patch | 12 + repo/umpire/package.py | 376 ++++++++++++++++++ repo/umpire/std-filesystem-pr784.patch | 33 ++ 14 files changed, 984 insertions(+), 73 deletions(-) create mode 100644 repo/blt/package.py create mode 100644 repo/camp/libstdc++-13-missing-header.patch create mode 100644 repo/camp/package.py create mode 100644 repo/chai/package.py create mode 100644 repo/raja/package.py create mode 100644 repo/umpire/camp_target_umpire_3.0.0.patch create mode 100644 repo/umpire/cmake_version_check.patch create mode 100644 repo/umpire/missing_header_for_numeric_limits.patch create mode 100644 repo/umpire/package.py create mode 100644 repo/umpire/std-filesystem-pr784.patch diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/openmp/execute_experiment.tpl +++ b/experiments/kripke/openmp/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml index 0ede5c91f..670af7156 100644 --- a/experiments/kripke/openmp/ramble.yaml +++ b/experiments/kripke/openmp/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,41 +18,40 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: set: OMP_NUM_THREADS: '{omp_num_threads}' variables: n_ranks: '{processes_per_node} * {n_nodes}' - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' experiment_setup: '' - processes_per_node: ['8', '4'] - n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks - omp_num_threads: '{threads_per_node_core} * {n_nodes}' + processes_per_node: ['1', '16'] + n_nodes: '1' + omp_num_threads: ['32', '2'] + arch: 'OpenMP' experiments: - kripke_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_threads: - - n # TODO: Filter matrix - - threads_per_node_core # TODO: Filter matrix + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '32' + nzz: '32' + npx: ['1', '4'] + npy: ['1', '2'] + npz: ['1', '2'] spack: concretized: true packages: - kripke-omp: - spack_spec: kripke@develop +mpi+openmp + kripke: + spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-omp: + kripke: packages: - default-mpi - - kripke-omp + - kripke + - '{modifier_package_name}' diff --git a/repo/blt/package.py b/repo/blt/package.py new file mode 100644 index 000000000..64725f485 --- /dev/null +++ b/repo/blt/package.py @@ -0,0 +1,11 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * +from spack.pkg.builtin.blt import Blt as BuiltinBlt + + +class Blt(BuiltinBlt): + version("0.6.2", sha256="84b663162957c1fe0e896ac8e94cbf2b6def4a152ccfa12a293db14fb25191c8") diff --git a/repo/camp/libstdc++-13-missing-header.patch b/repo/camp/libstdc++-13-missing-header.patch new file mode 100644 index 000000000..d3c642abe --- /dev/null +++ b/repo/camp/libstdc++-13-missing-header.patch @@ -0,0 +1,24 @@ +diff --git a/include/camp/resource.hpp b/include/camp/resource.hpp +index 58df206..763aa20 100644 +--- a/include/camp/resource.hpp ++++ b/include/camp/resource.hpp +@@ -12,6 +12,7 @@ http://github.com/llnl/camp + #define __CAMP_RESOURCE_HPP + + #include ++#include + #include + #include + #include +diff --git a/include/camp/resource/host.hpp b/include/camp/resource/host.hpp +index c765175..4428c57 100644 +--- a/include/camp/resource/host.hpp ++++ b/include/camp/resource/host.hpp +@@ -13,6 +13,7 @@ http://github.com/llnl/camp + + #include "camp/resource/event.hpp" + #include "camp/resource/platform.hpp" ++#include + + namespace camp + { diff --git a/repo/camp/package.py b/repo/camp/package.py new file mode 100644 index 000000000..241648029 --- /dev/null +++ b/repo/camp/package.py @@ -0,0 +1,108 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import glob + +from spack.package import * + + +def hip_repair_options(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + "-DHIP_CLANG_INCLUDE_PATH=" + + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0] + ) + + +def hip_repair_cache(options, spec): + # there is only one dir like this, but the version component is unknown + options.append( + cmake_cache_path( + "HIP_CLANG_INCLUDE_PATH", + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0], + ) + ) + + +class Camp(CMakePackage, CudaPackage, ROCmPackage): + """ + Compiler agnostic metaprogramming library providing concepts, + type operations and tuples for C++ and cuda + """ + + homepage = "https://github.com/LLNL/camp" + git = "https://github.com/LLNL/camp.git" + url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" + + maintainers("trws") + + license("BSD-3-Clause") + + version("main", branch="main", submodules="True") + version( + "2024.02.1", + tag="v2024.02.1", + commit="79c320fa09db987923b56884afdc9f82f4b70fc4", + submodules=False, + ) + version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02") + version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") + version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") + version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") + version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") + version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") + version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") + + # TODO: figure out gtest dependency and then set this default True. + variant("tests", default=False, description="Build tests") + variant("openmp", default=False, description="Build OpenMP support") + + depends_on("cub", when="+cuda") + + depends_on("blt", type="build") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2024.01") + depends_on("blt@0.6.2", type="build", when="@2024.02:") + + patch("libstdc++-13-missing-header.patch", when="@:2022.10") + + conflicts("^blt@:0.3.6", when="+rocm") + + def cmake_args(self): + spec = self.spec + + options = [] + + options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) + + options.append("-DENABLE_OPENMP=" + ("On" if "+openmp" in spec else "Off")) + if "+cuda" in spec: + options.extend( + ["-DENABLE_CUDA=ON", "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix)] + ) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) + options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) + flag = "-arch sm_{0}".format(cuda_arch[0]) + options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) + else: + options.append("-DENABLE_CUDA=OFF") + + if "+rocm" in spec: + options.extend(["-DENABLE_HIP=ON", "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)]) + + hip_repair_options(options, spec) + + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str)) + else: + options.append("-DENABLE_HIP=OFF") + + options.append(self.define_from_variant("ENABLE_TESTS", "tests")) + + return options diff --git a/repo/chai/package.py b/repo/chai/package.py new file mode 100644 index 000000000..152d94dcc --- /dev/null +++ b/repo/chai/package.py @@ -0,0 +1,228 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import socket + +from spack.package import * + + +class Chai(CachedCMakePackage, CudaPackage, ROCmPackage): + """ + Copy-hiding array interface for data migration between memory spaces + """ + + homepage = "https://github.com/LLNL/CHAI" + git = "https://github.com/LLNL/CHAI.git" + tags = ["ecp", "e4s", "radiuss"] + + maintainers("davidbeckingsale") + + license("BSD-3-Clause") + + version("develop", branch="develop", submodules=False) + version("main", branch="main", submodules=False) + version( + "2024.02.0", + tag="v2024.02.0", + commit="31773a2f0d30f3f64c82939f60fc4da32cf33261", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="f0b809de1ac194376866b3ac0f5933d4146ec09e", + submodules=False, + ) + version( + "2.4.0", tag="v2.4.0", commit="77d22da28187245a2c5454cf471c0c352bd98ad7", submodules=True + ) + version( + "2.3.0", tag="v2.3.0", commit="42f3fbcc0b966227b40b4467dc919a4c24f07196", submodules=True + ) + version( + "2.2.2", tag="v2.2.2", commit="56e75fc0f805b2746f3992af0c00c474513e3b24", submodules=True + ) + version( + "2.2.1", tag="v2.2.1", commit="c912f583828ea5963850816e3e232cc45608ccf7", submodules=True + ) + version( + "2.2.0", tag="v2.2.0", commit="18536c61a4817db6b3b3025f35e2dd3ae532330c", submodules=True + ) + version( + "2.1.1", tag="v2.1.1", commit="496911e00d15c350560860f7964cd5fb5ab7f515", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="fff02768068a64970b34760a1041585319edee87", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="63139cf45443b1266950826b165e042c7679b557", submodules=True + ) + version( + "1.2.0", tag="v1.2.0", commit="7bb5bc12e4508db45910d8e2b98444687da7ebf6", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True + ) + version("1.0", tag="v1.0", commit="501a098ad879dc8deb4a74fcfe8c08c283a10627", submodules=True) + + variant("enable_pick", default=False, description="Enable pick method") + variant( + "separable_compilation", + default=True, + description="Build with CUDA_SEPARABLE_COMPILATION flag on ", + ) + variant("shared", default=True, description="Build Shared Libs") + variant("raja", default=False, description="Build plugin for RAJA") + variant("benchmarks", default=False, description="Build benchmarks.") + variant("examples", default=True, description="Build examples.") + variant("openmp", default=False, description="Build using OpenMP") + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant("tests", default=False, description="Build tests") + + depends_on("cmake@3.8:", type="build") + depends_on("cmake@3.9:", type="build", when="+cuda") + depends_on("cmake@3.14:", when="@2022.03.0:") + + depends_on("blt@0.6.2", type="build", when="@2024.02:") + depends_on("blt@0.5.0:", type="build", when="@2022.03.0:2024.01") + depends_on("blt@0.4.1:", type="build", when="@2.4.0:") + depends_on("blt@0.4.0:", type="build", when="@2.3.0") + depends_on("blt@0.3.6:", type="build", when="@:2.2.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("umpire") + depends_on("umpire@2024.02:", when="@2024.02:") + depends_on("umpire@2022.03.0:", when="@2022.03.0:2024.01") + depends_on("umpire@6.0.0", when="@2.4.0") + depends_on("umpire@4.1.2", when="@2.2.0:2.3.0") + depends_on("umpire@main", when="@main") + + with when("+cuda"): + depends_on("umpire+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("umpire+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm"): + depends_on("umpire+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "umpire+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) + ) + + with when("+raja"): + depends_on("raja~openmp", when="~openmp") + depends_on("raja+openmp", when="+openmp") + depends_on("raja@0.14.0", when="@2.4.0") + depends_on("raja@0.13.0", when="@2.3.0") + depends_on("raja@0.12.0", when="@2.2.0:2.2.2") + depends_on("raja@2022.03.0:", when="@2022.03.0:") + depends_on("raja@main", when="@main") + + with when("+cuda"): + depends_on("raja+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("raja+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + with when("+rocm"): + depends_on("raja+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "raja+rocm amdgpu_target={0}".format(arch), + when="amdgpu_target={0}".format(arch), + ) + + conflicts("+benchmarks", when="~tests") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + ) + + def initconfig_compiler_entries(self): + spec = self.spec + entries = super().initconfig_compiler_entries() + if "+rocm" in spec: + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) + + if "+cuda" in spec: + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + if "+separable_compilation" in spec: + entries.append(cmake_cache_option("CMAKE_CUDA_SEPARABLE_COMPILATION", True)) + entries.append(cmake_cache_option("CUDA_SEPARABLE_COMPILATION", True)) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) + entries.append( + cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) + ) + flag = "-arch sm_{0}".format(cuda_arch[0]) + entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if "+rocm" in spec: + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append( + cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) + ) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "CHAI_" if spec.satisfies("@2022.03.0:") else "" + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if "+raja" in spec: + entries.append(cmake_cache_option("{}ENABLE_RAJA_PLUGIN".format(option_prefix), True)) + entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) + entries.append( + cmake_cache_option("{}ENABLE_PICK".format(option_prefix), "+enable_pick" in spec) + ) + if spec.satisfies("@2024.02:"): + entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.lib64.cmake.umpire)) + else: + entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.share.umpire.cmake)) + entries.append(cmake_cache_option("ENABLE_TESTS", "+tests" in spec)) + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "+benchmarks" in spec)) + entries.append( + cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) + ) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) + + return entries + + def cmake_args(self): + options = [] + return options diff --git a/repo/kripke/application.py b/repo/kripke/application.py index e9d0ba2a3..61a79f7f6 100644 --- a/repo/kripke/application.py +++ b/repo/kripke/application.py @@ -17,44 +17,102 @@ class Kripke(SpackApplication): 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', 'mpi','network-latency-bound','network-collectives'] - executable('p1', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 1' + - ' -keepT', use_mpi=True) - - executable('p2', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 2' + - ' -keepT', use_mpi=True) - - workload('problem1', executables=['p1']) - workload('problem2', executables=['p2']) - - workload_variable('px', default='2', - description='px', - workloads=['problem1', 'problem2']) - workload_variable('py', default='2', - description='py', - workloads=['problem1', 'problem2']) - workload_variable('pz', default='2', - description='pz', - workloads=['problem1', 'problem2']) - workload_variable('nx', default='220', - description='nx', - workloads=['problem1', 'problem2']) - workload_variable('ny', default='220', - description='ny', - workloads=['problem1', 'problem2']) - workload_variable('nz', default='220', - description='nz', - workloads=['problem1', 'problem2']) - - figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + executable('kripke', 'kripke.exe' + + ' --groups {ngroups}' + + ' --legendre {lorder}' + + ' --quad {nquad}' + + ' --zones {nzx},{nzy},{nzz}' + + ' --sigt {sigt0},{sigt1},{sigt2}' + + ' --sigs {sigs0},{sigs1},{sigs2}' + + ' --arch {arch}' + + ' --layout {layout}' + +# ' --pdist {lout}' + + ' --procs {npx},{npy},{npz}' + + ' --dset {ds}' + + ' --gset {gs}' + + ' --zset {nzsetx},{nzsety},{nzsetz}' + + ' --niter {niter}' + + ' --pmethod {method}', use_mpi=True) - #TODO: Fix the FOM success_criteria(...) - success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') + workload('kripke', executables=['kripke']) + + workload_variable('ngroups', default='32', + description='Number of energy groups. (Default: --groups 32)', + workloads=['kripke']) + workload_variable('lorder', default='4', + description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', + workloads=['kripke']) + workload_variable('nquad', default='96', + description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in x. (Default: 16)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in y. (Default: 16)', + workloads=['kripke']) + workload_variable('nzz', default='16', + description='Number of zones in z. (Default: 16)', + workloads=['kripke']) + workload_variable('sigt0', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt1', default='0.0001', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt2', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs0', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs1', default='0.00005', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs2', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('arch', default='OpenMP', + description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', + workloads=['kripke']) + workload_variable('layout', default='DGZ', + description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', + workloads=['kripke']) +# workload_variable('lout', default='0', +# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', +# workloads=['kripke']) + workload_variable('npx', default='1', + description='Number of MPI ranks in x dimension', + workloads=['kripke']) + workload_variable('npy', default='1', + description='Number of MPI ranks in y dimension', + workloads=['kripke']) + workload_variable('npz', default='1', + description='Number of MPI ranks in z dimension', + workloads=['kripke']) + workload_variable('ds', default='8', + description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', + workloads=['kripke']) + workload_variable('gs', default='1', + description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', + workloads=['kripke']) + workload_variable('nzsetx', default='1', + description='Number of zone-sets in x', + workloads=['kripke']) + workload_variable('nzsety', default='1', + description='Number of zone-sets in y', + workloads=['kripke']) + workload_variable('nzsetz', default='1', + description='Number of zone-sets in z', + workloads=['kripke']) + workload_variable('niter', default='10', + description='Number of solver iterations to run. (Default: --niter 10)', + workloads=['kripke']) + workload_variable('method', default='sweep', + description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', + workloads=['kripke']) - def evaluate_success(self): - return True + #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + + #TODO: Fix the FOM success_criteria(...) + #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 6bf335f35..7aed1369e 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -56,17 +56,32 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") + depends_on("blt@0.6.2", type="build") + conflicts("^blt@:0.3.6", when="+rocm") + aligned_versions = ["2024.02"] - constrain_chai = list(f"^chai@{v}" for v in aligned_versions) - requires(*aligned_versions, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") - + #constrain_chai = list(f"^chai@{v}" for v in aligned_versions) + #requires(*constrain_chai, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") + for v in aligned_versions: depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") depends_on(f"umpire@{v}~examples", when=f"^chai@{v}") depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}") - - depends_on("blt@0.6.2", type="build") - conflicts("^blt@:0.3.6", when="+rocm") + depends_on(f"camp@{v}", when=f"^chai@{v}") + + depends_on("chai+openmp", when="+openmp") + depends_on("chai~openmp", when="~openmp") + depends_on("chai+cuda", when="+cuda") + depends_on("chai~cuda", when="~cuda") + depends_on("chai+rocm", when="+rocm") + depends_on("chai~rocm", when="~rocm") + + depends_on("umpire+openmp", when="+openmp") + depends_on("umpire~openmp", when="~openmp") + depends_on("umpire+cuda", when="+cuda") + depends_on("umpire~cuda", when="~cuda") + depends_on("umpire+rocm", when="+rocm") + depends_on("umpire~rocm", when="~rocm") def cmake_args(self): spec = self.spec @@ -83,6 +98,9 @@ def cmake_args(self): ] ) + if "+openmp" in spec: + args.append("-DENABLE_OPENMP=ON") + if "+caliper" in spec: args.append("-DENABLE_CALIPER=ON") diff --git a/repo/raja/package.py b/repo/raja/package.py new file mode 100644 index 000000000..41e35ac95 --- /dev/null +++ b/repo/raja/package.py @@ -0,0 +1,17 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * +from spack.pkg.builtin.raja import Raja as BuiltinRaja + + +class Raja(BuiltinRaja): + version( + "2024.02.1", + tag="v2024.02.1", + commit="3ada0950b0774ec907d30a9eceaf6af7478b833b", + submodules=False, + ) + depends_on("blt@0.6.2", type="build", when="@2024.02:") diff --git a/repo/umpire/camp_target_umpire_3.0.0.patch b/repo/umpire/camp_target_umpire_3.0.0.patch new file mode 100644 index 000000000..2deb565cd --- /dev/null +++ b/repo/umpire/camp_target_umpire_3.0.0.patch @@ -0,0 +1,18 @@ +diff --git a/umpire-config.cmake.in b/umpire-config.cmake.in +index a98ad5fe..4e54e173 100644 +--- a/umpire-config.cmake.in ++++ b/umpire-config.cmake.in +@@ -7,6 +7,13 @@ + get_filename_component(UMPIRE_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) + set(UMPIRE_INCLUDE_DIRS "@CMAKE_INSTALL_PREFIX@/include") + ++if (NOT TARGET camp) ++ if (NOT DEFINED camp_DIR) ++ set(camp_DIR @CMAKE_INSTALL_PREFIX@/lib/cmake/camp) ++ endif () ++ find_package(camp REQUIRED) ++endif () ++ + set(Umpire_VERSION_MAJOR @Umpire_VERSION_MAJOR@) + set(Umpire_VERSION_MINOR @Umpire_VERSION_MINOR@) + set(Umpire_VERSION_PATCH @Umpire_VERSION_PATCH@) diff --git a/repo/umpire/cmake_version_check.patch b/repo/umpire/cmake_version_check.patch new file mode 100644 index 000000000..05f80e0b6 --- /dev/null +++ b/repo/umpire/cmake_version_check.patch @@ -0,0 +1,13 @@ +diff --git a/src/umpire/CMakeLists.txt b/src/umpire/CMakeLists.txt +index 18518f13..c2e45e03 100644 +--- a/src/umpire/CMakeLists.txt ++++ b/src/umpire/CMakeLists.txt +@@ -105,7 +105,7 @@ blt_add_library( + DEPENDS_ON ${umpire_depends} + DEFINES ${umpire_defines}) + +-if (NOT WIN32 AND CMAKE_VERSION VERSION_GREATER 3.17) ++if (NOT WIN32 AND NOT ${CMAKE_VERSION} VERSION_LESS 3.18) + target_link_options( + umpire INTERFACE + $>:-rdynamic>> diff --git a/repo/umpire/missing_header_for_numeric_limits.patch b/repo/umpire/missing_header_for_numeric_limits.patch new file mode 100644 index 000000000..6e89e3707 --- /dev/null +++ b/repo/umpire/missing_header_for_numeric_limits.patch @@ -0,0 +1,12 @@ +diff --git a/src/umpire/util/allocation_statistics.cpp b/src/umpire/util/allocation_statistics.cpp +index 597ae61f..0c3f7865 100644 +--- a/src/umpire/util/allocation_statistics.cpp ++++ b/src/umpire/util/allocation_statistics.cpp +@@ -8,6 +8,7 @@ + #include "umpire/util/allocation_statistics.hpp" + + #include ++#include + + namespace umpire { + namespace util { diff --git a/repo/umpire/package.py b/repo/umpire/package.py new file mode 100644 index 000000000..7c23151c4 --- /dev/null +++ b/repo/umpire/package.py @@ -0,0 +1,376 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import socket + +import llnl.util.tty as tty + +from spack.package import * +from spack.pkg.builtin.camp import hip_repair_cache + + +class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): + """An application-focused API for memory management on NUMA & GPU + architectures""" + + homepage = "https://github.com/LLNL/Umpire" + git = "https://github.com/LLNL/Umpire.git" + tags = ["radiuss", "e4s"] + + maintainers("davidbeckingsale") + + license("MIT") + + version("develop", branch="develop", submodules=False) + version("main", branch="main", submodules=False) + version( + "2024.02.1", + tag="v2024.02.1", + commit="3058d562fc707650e904f9321b1ee9bcebad3ae2", + submodules=True, + ) + version( + "2022.10.0", + tag="v2022.10.0", + commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a", + submodules=False, + ) + version( + "2022.03.1", + tag="v2022.03.1", + commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b", + submodules=False, + ) + version( + "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True + ) + version( + "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True + ) + version( + "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True + ) + version( + "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True + ) + version( + "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True + ) + version( + "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True + ) + version( + "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True + ) + version( + "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True + ) + version( + "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True + ) + version( + "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True + ) + version( + "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True + ) + version( + "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True + ) + version( + "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True + ) + version( + "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True + ) + version( + "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True + ) + version( + "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True + ) + version( + "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True + ) + version( + "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True + ) + version( + "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True + ) + version( + "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True + ) + version( + "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True + ) + version( + "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True + ) + version( + "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True + ) + version( + "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True + ) + + patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:") + patch("camp_target_umpire_3.0.0.patch", when="@3.0.0") + patch("cmake_version_check.patch", when="@4.1") + patch("missing_header_for_numeric_limits.patch", when="@4.1:5.0.1") + + # export targets when building pre-6.0.0 release with BLT 0.4.0+ + patch( + "https://github.com/LLNL/Umpire/commit/5773ce9af88952c8d23f9bcdcb2e503ceda40763.patch?full_index=1", + sha256="f3b21335ce5cf9c0fecc852a94dfec90fb5703032ac97f9fee104af9408d8899", + when="@:5.0.1 ^blt@0.4:", + ) + + # https://github.com/LLNL/Umpire/pull/805 + patch( + "https://github.com/LLNL/Umpire/pull/805/commits/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1", + sha256="7ed5d2c315a3b31e339f664f6108e32d7cb4cb8e9f22e5c78a65ba02625ccc09", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/816 + patch( + "https://github.com/LLNL/Umpire/pull/816/commits/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1", + sha256="0f43cad7cdaec3c225ab6414ab9f81bd405a1157abf5a508e515bcb6ca53326d", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/853 + patch( + "https://github.com/LLNL/Umpire/commit/4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256.patch?full_index=1", + sha256="c9ddae1f4212cef72e1050b6ac482ce5b795dad4977d2462cff2e884b8d7aff5", + when="@2022.10:2023.06", + ) + + variant("fortran", default=False, description="Build C/Fortran API") + variant("c", default=True, description="Build C API") + variant("numa", default=False, description="Enable NUMA support") + variant("shared", default=True, description="Enable Shared libs") + variant("openmp", default=False, description="Build with OpenMP support") + variant("deviceconst", default=False, description="Enables support for constant device memory") + variant("examples", default=True, description="Build Umpire Examples") + variant( + "tests", + default="none", + values=("none", "basic", "benchmarks"), + multi=False, + description="Tests to run", + ) + variant("device_alloc", default=True, description="Build Umpire Device Allocator") + + depends_on("cmake@3.8:", type="build") + depends_on("cmake@3.9:", when="+cuda", type="build") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") + + depends_on("blt", type="build") + depends_on("blt@0.6.2", type="build", when="@2024.02:") + depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2022.03.1") + depends_on("blt@0.4.1", type="build", when="@6.0.0") + depends_on("blt@0.4.0:", type="build", when="@4.1.3:5.0.1") + depends_on("blt@0.3.6:", type="build", when="@:4.1.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("camp", when="@5.0.0:") + depends_on("camp@0.2.2:0.2.3", when="@6.0.0") + depends_on("camp@0.1.0", when="@5.0.0:5.0.1") + depends_on("camp@2022.03.2:", when="@2022.03.0:2024.01") + depends_on("camp@2024.02:", when="@2024.02:") + depends_on("camp@main", when="@main") + depends_on("camp@main", when="@develop") + depends_on("camp+openmp", when="+openmp") + depends_on("camp~cuda", when="~cuda") + depends_on("camp~rocm", when="~rocm") + + with when("@5.0.0:"): + with when("+cuda"): + depends_on("camp+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("camp+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm"): + depends_on("camp+rocm") + for arch_ in ROCmPackage.amdgpu_targets: + depends_on( + "camp+rocm amdgpu_target={0}".format(arch_), + when="amdgpu_target={0}".format(arch_), + ) + + conflicts("+numa", when="@:0.3.2") + conflicts("~c", when="+fortran", msg="Fortran API requires C API") + conflicts("+device_alloc", when="@:2022.03.0") + + # device allocator exports device code, which requires static libs + # currently only available for cuda. + conflicts("+shared", when="+cuda") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + ) + + def initconfig_compiler_entries(self): + spec = self.spec + entries = super().initconfig_compiler_entries() + + if "+rocm" in spec: + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if "+fortran" in spec and self.compiler.fc is not None: + entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) + else: + entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) + + entries.append(cmake_cache_option("{}ENABLE_C".format(option_prefix), "+c" in spec)) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if "+cuda" in spec: + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) + entries.append( + cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) + ) + flag = "-arch sm_{0}".format(cuda_arch[0]) + entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) + + entries.append( + cmake_cache_option( + "{}ENABLE_DEVICE_CONST".format(option_prefix), spec.satisfies("+deviceconst") + ) + ) + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if "+rocm" in spec: + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) + hip_repair_cache(entries, spec) + archs = self.spec.variants["amdgpu_target"].value + if archs != "none": + arch_str = ",".join(archs) + entries.append( + cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) + ) + entries.append( + cmake_cache_string("CMAKE_HIP_ARCHITECTURES", "{0}".format(arch_str)) + ) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if spec.satisfies("@5.0.0:"): + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + entries.append(cmake_cache_option("{}ENABLE_NUMA".format(option_prefix), "+numa" in spec)) + entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "tests=benchmarks" in spec)) + entries.append( + cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) + ) + entries.append(cmake_cache_option("{}ENABLE_DOCS".format(option_prefix), False)) + entries.append( + cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", "+device_alloc" in spec) + ) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) + entries.append(cmake_cache_option("ENABLE_TESTS", "tests=none" not in spec)) + + return entries + + def cmake_args(self): + options = [] + return options + + def test(self): + """Perform stand-alone checks on the installed package.""" + if self.spec.satisfies("@:1") or not os.path.isdir(self.prefix.bin): + tty.info("Skipping: checks not installed in bin for v{0}".format(self.version)) + return + + # Run a subset of examples PROVIDED installed + # tutorials with readily checkable outputs. + checks = { + "malloc": ["99 should be 99"], + "recipe_dynamic_pool_heuristic": ["in the pool", "releas"], + "recipe_no_introspection": ["has allocated", "used"], + "strategy_example": ["Available allocators", "HOST"], + "tut_copy": ["Copied source data"], + "tut_introspection": ["Allocator used is HOST", "size of the allocation"], + "tut_memset": ["Set data from HOST"], + "tut_move": ["Moved source data", "HOST"], + "tut_reallocate": ["Reallocated data"], + "vector_allocator": [""], + } + + for exe in checks: + expected = checks[exe] + reason = "test: checking output from {0}".format(exe) + self.run_test( + exe, + [], + expected, + 0, + installed=False, + purpose=reason, + skip_missing=True, + work_dir=self.prefix.bin, + ) diff --git a/repo/umpire/std-filesystem-pr784.patch b/repo/umpire/std-filesystem-pr784.patch new file mode 100644 index 000000000..677144214 --- /dev/null +++ b/repo/umpire/std-filesystem-pr784.patch @@ -0,0 +1,33 @@ +diff -ruN spack-src/cmake/SetupCompilerFlags.cmake spack-src-patched/cmake/SetupCompilerFlags.cmake +--- spack-src/cmake/SetupCompilerFlags.cmake 2022-10-12 08:05:03.538390165 -0700 ++++ spack-src-patched/cmake/SetupCompilerFlags.cmake 2022-10-12 09:47:56.317645003 -0700 +@@ -11,20 +11,20 @@ + + message(STATUS "Checking for std::filesystem") + +-include(CheckCXXSourceCompiles) +-check_cxx_source_compiles( +- "#include +- #include +- +- int main(int, char**) +- { ++blt_check_code_compiles(CODE_COMPILES UMPIRE_ENABLE_FILESYSTEM ++ SOURCE_STRING ++[=[ ++#include ++#include + ++int main(int, char**) ++{ + auto path = std::filesystem::path(\".\"); + (void)(path); + + return 0; +- }" +- UMPIRE_ENABLE_FILESYSTEM) ++} ++]=]) + + if (UMPIRE_ENABLE_FILESYSTEM) + message(STATUS "std::filesystem found") From 05d36def4fd01d567b187a0e75e885071509f01a Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Mon, 29 Apr 2024 01:11:15 -0700 Subject: [PATCH 02/14] HIP implementation of kripke --- .../kripke/rocm/execute_experiment.tpl | 5 --- experiments/kripke/rocm/ramble.yaml | 45 ++++++++++--------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/experiments/kripke/rocm/execute_experiment.tpl b/experiments/kripke/rocm/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/rocm/execute_experiment.tpl +++ b/experiments/kripke/rocm/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml index d22da9f3d..1085636ac 100644 --- a/experiments/kripke/rocm/ramble.yaml +++ b/experiments/kripke/rocm/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,37 +18,37 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: variables: - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' - experiment_setup: '' - processes_per_node: ['4'] - n_nodes: ['1', '2'] n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: ['8'] + n_nodes: '1' + arch: 'HIP' experiments: - kripke_rocm_problem1_{n_nodes}_{n_ranks}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_rocm_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_jobs: - - n # TODO: Filter matrix - - n_nodes # TODO: Filter matrix + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '64' + npx: '2' + npy: '2' + npz: '2' spack: concretized: true packages: - kripke-rocm: - spack_spec: kripke@updateRCU2024.02 +mpi+rocm + kripke: + spack_spec: kripke@develop +mpi+rocm{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-rocm: + kripke: packages: - default-mpi - - kripke-rocm + - kripke + - '{modifier_package_name}' From 5a97c41e53d2652a1795620597e03a97c734d1dd Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Mon, 29 Apr 2024 15:05:45 -0700 Subject: [PATCH 03/14] Update spack version. Pin kripke version --- checkout-versions.yaml | 2 +- .../auxiliary_software_files/packages.yaml | 2 +- .../kripke/cuda/execute_experiment.tpl | 5 - experiments/kripke/cuda/ramble.yaml | 44 +- experiments/kripke/openmp/ramble.yaml | 2 +- experiments/kripke/rocm/ramble.yaml | 2 +- repo/blt/package.py | 11 - repo/camp/libstdc++-13-missing-header.patch | 24 -- repo/camp/package.py | 108 ----- repo/chai/package.py | 228 ----------- repo/kripke/.package.py.swp | Bin 0 -> 16384 bytes repo/kripke/package.py | 3 + repo/raja/package.py | 17 - repo/umpire/camp_target_umpire_3.0.0.patch | 18 - repo/umpire/cmake_version_check.patch | 13 - .../missing_header_for_numeric_limits.patch | 12 - repo/umpire/package.py | 376 ------------------ repo/umpire/std-filesystem-pr784.patch | 33 -- 18 files changed, 29 insertions(+), 871 deletions(-) delete mode 100644 repo/blt/package.py delete mode 100644 repo/camp/libstdc++-13-missing-header.patch delete mode 100644 repo/camp/package.py delete mode 100644 repo/chai/package.py create mode 100644 repo/kripke/.package.py.swp delete mode 100644 repo/raja/package.py delete mode 100644 repo/umpire/camp_target_umpire_3.0.0.patch delete mode 100644 repo/umpire/cmake_version_check.patch delete mode 100644 repo/umpire/missing_header_for_numeric_limits.patch delete mode 100644 repo/umpire/package.py delete mode 100644 repo/umpire/std-filesystem-pr784.patch diff --git a/checkout-versions.yaml b/checkout-versions.yaml index 2d7d07ce6..6c3973512 100644 --- a/checkout-versions.yaml +++ b/checkout-versions.yaml @@ -5,4 +5,4 @@ versions: ramble: bb664f142b2cbdb2b2ea39e70a8535c9f27c1179 - spack: 31de670bd26beca979ebd75dcb0ce90c535a78c4 + spack: c2eef8bab26adb00b250992e29d697b4706356a0 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index 1b64a0ee3..a13d6f3b3 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -18,7 +18,7 @@ packages: externals: - spec: cmake@3.23.1 prefix: /usr/tce/packages/cmake/cmake-3.23.1 - buildable: false + buildable: false gmake: externals: - spec: gmake@4.2.1 diff --git a/experiments/kripke/cuda/execute_experiment.tpl b/experiments/kripke/cuda/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/cuda/execute_experiment.tpl +++ b/experiments/kripke/cuda/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml index 0eec22b58..25cc12855 100644 --- a/experiments/kripke/cuda/ramble.yaml +++ b/experiments/kripke/cuda/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,38 +18,37 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: variables: - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' + n_ranks: '{processes_per_node} * {n_nodes}' experiment_setup: '' processes_per_node: ['4'] - n_nodes: ['1', '2'] - n_ranks: '{processes_per_node} * {n_nodes}' + n_nodes: '1' + arch: 'CUDA' experiments: - kripke_rocm_problem1_{n_nodes}_{n_ranks}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_jobs: - - n # TODO: Filter matrix - - n_nodes # TODO: Filter matrix - + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '32' + npx: '2' + npy: '2' + npz: '1' spack: concretized: true packages: - kripke-cuda: - spack_spec: kripke@updateRCU2024.02 +cuda + kripke: + spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-cuda: + kripke: packages: - default-mpi - - kripke-cuda + - kripke + - '{modifier_package_name}' diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml index 670af7156..afdbaa127 100644 --- a/experiments/kripke/openmp/ramble.yaml +++ b/experiments/kripke/openmp/ramble.yaml @@ -47,7 +47,7 @@ ramble: concretized: true packages: kripke: - spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^chai@2024.02 + spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: kripke: diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml index 1085636ac..7439b57dc 100644 --- a/experiments/kripke/rocm/ramble.yaml +++ b/experiments/kripke/rocm/ramble.yaml @@ -44,7 +44,7 @@ ramble: concretized: true packages: kripke: - spack_spec: kripke@develop +mpi+rocm{modifier_spack_variant} ^chai@2024.02 + spack_spec: kripke@1.2.7.0 +mpi+rocm{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: kripke: diff --git a/repo/blt/package.py b/repo/blt/package.py deleted file mode 100644 index 64725f485..000000000 --- a/repo/blt/package.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from spack.package import * -from spack.pkg.builtin.blt import Blt as BuiltinBlt - - -class Blt(BuiltinBlt): - version("0.6.2", sha256="84b663162957c1fe0e896ac8e94cbf2b6def4a152ccfa12a293db14fb25191c8") diff --git a/repo/camp/libstdc++-13-missing-header.patch b/repo/camp/libstdc++-13-missing-header.patch deleted file mode 100644 index d3c642abe..000000000 --- a/repo/camp/libstdc++-13-missing-header.patch +++ /dev/null @@ -1,24 +0,0 @@ -diff --git a/include/camp/resource.hpp b/include/camp/resource.hpp -index 58df206..763aa20 100644 ---- a/include/camp/resource.hpp -+++ b/include/camp/resource.hpp -@@ -12,6 +12,7 @@ http://github.com/llnl/camp - #define __CAMP_RESOURCE_HPP - - #include -+#include - #include - #include - #include -diff --git a/include/camp/resource/host.hpp b/include/camp/resource/host.hpp -index c765175..4428c57 100644 ---- a/include/camp/resource/host.hpp -+++ b/include/camp/resource/host.hpp -@@ -13,6 +13,7 @@ http://github.com/llnl/camp - - #include "camp/resource/event.hpp" - #include "camp/resource/platform.hpp" -+#include - - namespace camp - { diff --git a/repo/camp/package.py b/repo/camp/package.py deleted file mode 100644 index 241648029..000000000 --- a/repo/camp/package.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import glob - -from spack.package import * - - -def hip_repair_options(options, spec): - # there is only one dir like this, but the version component is unknown - options.append( - "-DHIP_CLANG_INCLUDE_PATH=" - + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0] - ) - - -def hip_repair_cache(options, spec): - # there is only one dir like this, but the version component is unknown - options.append( - cmake_cache_path( - "HIP_CLANG_INCLUDE_PATH", - glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0], - ) - ) - - -class Camp(CMakePackage, CudaPackage, ROCmPackage): - """ - Compiler agnostic metaprogramming library providing concepts, - type operations and tuples for C++ and cuda - """ - - homepage = "https://github.com/LLNL/camp" - git = "https://github.com/LLNL/camp.git" - url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" - - maintainers("trws") - - license("BSD-3-Clause") - - version("main", branch="main", submodules="True") - version( - "2024.02.1", - tag="v2024.02.1", - commit="79c320fa09db987923b56884afdc9f82f4b70fc4", - submodules=False, - ) - version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02") - version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") - version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") - version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") - version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") - version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") - version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") - - # TODO: figure out gtest dependency and then set this default True. - variant("tests", default=False, description="Build tests") - variant("openmp", default=False, description="Build OpenMP support") - - depends_on("cub", when="+cuda") - - depends_on("blt", type="build") - depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2024.01") - depends_on("blt@0.6.2", type="build", when="@2024.02:") - - patch("libstdc++-13-missing-header.patch", when="@:2022.10") - - conflicts("^blt@:0.3.6", when="+rocm") - - def cmake_args(self): - spec = self.spec - - options = [] - - options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) - - options.append("-DENABLE_OPENMP=" + ("On" if "+openmp" in spec else "Off")) - if "+cuda" in spec: - options.extend( - ["-DENABLE_CUDA=ON", "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix)] - ) - - if not spec.satisfies("cuda_arch=none"): - cuda_arch = spec.variants["cuda_arch"].value - options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) - options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) - flag = "-arch sm_{0}".format(cuda_arch[0]) - options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) - else: - options.append("-DENABLE_CUDA=OFF") - - if "+rocm" in spec: - options.extend(["-DENABLE_HIP=ON", "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)]) - - hip_repair_options(options, spec) - - archs = self.spec.variants["amdgpu_target"].value - if archs != "none": - arch_str = ",".join(archs) - options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str)) - else: - options.append("-DENABLE_HIP=OFF") - - options.append(self.define_from_variant("ENABLE_TESTS", "tests")) - - return options diff --git a/repo/chai/package.py b/repo/chai/package.py deleted file mode 100644 index 152d94dcc..000000000 --- a/repo/chai/package.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import socket - -from spack.package import * - - -class Chai(CachedCMakePackage, CudaPackage, ROCmPackage): - """ - Copy-hiding array interface for data migration between memory spaces - """ - - homepage = "https://github.com/LLNL/CHAI" - git = "https://github.com/LLNL/CHAI.git" - tags = ["ecp", "e4s", "radiuss"] - - maintainers("davidbeckingsale") - - license("BSD-3-Clause") - - version("develop", branch="develop", submodules=False) - version("main", branch="main", submodules=False) - version( - "2024.02.0", - tag="v2024.02.0", - commit="31773a2f0d30f3f64c82939f60fc4da32cf33261", - submodules=False, - ) - version( - "2022.03.0", - tag="v2022.03.0", - commit="f0b809de1ac194376866b3ac0f5933d4146ec09e", - submodules=False, - ) - version( - "2.4.0", tag="v2.4.0", commit="77d22da28187245a2c5454cf471c0c352bd98ad7", submodules=True - ) - version( - "2.3.0", tag="v2.3.0", commit="42f3fbcc0b966227b40b4467dc919a4c24f07196", submodules=True - ) - version( - "2.2.2", tag="v2.2.2", commit="56e75fc0f805b2746f3992af0c00c474513e3b24", submodules=True - ) - version( - "2.2.1", tag="v2.2.1", commit="c912f583828ea5963850816e3e232cc45608ccf7", submodules=True - ) - version( - "2.2.0", tag="v2.2.0", commit="18536c61a4817db6b3b3025f35e2dd3ae532330c", submodules=True - ) - version( - "2.1.1", tag="v2.1.1", commit="496911e00d15c350560860f7964cd5fb5ab7f515", submodules=True - ) - version( - "2.1.0", tag="v2.1.0", commit="fff02768068a64970b34760a1041585319edee87", submodules=True - ) - version( - "2.0.0", tag="v2.0.0", commit="63139cf45443b1266950826b165e042c7679b557", submodules=True - ) - version( - "1.2.0", tag="v1.2.0", commit="7bb5bc12e4508db45910d8e2b98444687da7ebf6", submodules=True - ) - version( - "1.1.0", tag="v1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True - ) - version("1.0", tag="v1.0", commit="501a098ad879dc8deb4a74fcfe8c08c283a10627", submodules=True) - - variant("enable_pick", default=False, description="Enable pick method") - variant( - "separable_compilation", - default=True, - description="Build with CUDA_SEPARABLE_COMPILATION flag on ", - ) - variant("shared", default=True, description="Build Shared Libs") - variant("raja", default=False, description="Build plugin for RAJA") - variant("benchmarks", default=False, description="Build benchmarks.") - variant("examples", default=True, description="Build examples.") - variant("openmp", default=False, description="Build using OpenMP") - # TODO: figure out gtest dependency and then set this default True - # and remove the +tests conflict below. - variant("tests", default=False, description="Build tests") - - depends_on("cmake@3.8:", type="build") - depends_on("cmake@3.9:", type="build", when="+cuda") - depends_on("cmake@3.14:", when="@2022.03.0:") - - depends_on("blt@0.6.2", type="build", when="@2024.02:") - depends_on("blt@0.5.0:", type="build", when="@2022.03.0:2024.01") - depends_on("blt@0.4.1:", type="build", when="@2.4.0:") - depends_on("blt@0.4.0:", type="build", when="@2.3.0") - depends_on("blt@0.3.6:", type="build", when="@:2.2.2") - conflicts("^blt@:0.3.6", when="+rocm") - - depends_on("umpire") - depends_on("umpire@2024.02:", when="@2024.02:") - depends_on("umpire@2022.03.0:", when="@2022.03.0:2024.01") - depends_on("umpire@6.0.0", when="@2.4.0") - depends_on("umpire@4.1.2", when="@2.2.0:2.3.0") - depends_on("umpire@main", when="@main") - - with when("+cuda"): - depends_on("umpire+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("umpire+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - - with when("+rocm"): - depends_on("umpire+rocm") - for arch in ROCmPackage.amdgpu_targets: - depends_on( - "umpire+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) - ) - - with when("+raja"): - depends_on("raja~openmp", when="~openmp") - depends_on("raja+openmp", when="+openmp") - depends_on("raja@0.14.0", when="@2.4.0") - depends_on("raja@0.13.0", when="@2.3.0") - depends_on("raja@0.12.0", when="@2.2.0:2.2.2") - depends_on("raja@2022.03.0:", when="@2022.03.0:") - depends_on("raja@main", when="@main") - - with when("+cuda"): - depends_on("raja+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("raja+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - with when("+rocm"): - depends_on("raja+rocm") - for arch in ROCmPackage.amdgpu_targets: - depends_on( - "raja+rocm amdgpu_target={0}".format(arch), - when="amdgpu_target={0}".format(arch), - ) - - conflicts("+benchmarks", when="~tests") - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip("1234567890") - return "{0}-{1}-{2}@{3}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version, - ) - - def initconfig_compiler_entries(self): - spec = self.spec - entries = super().initconfig_compiler_entries() - if "+rocm" in spec: - entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) - return entries - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super().initconfig_hardware_entries() - - entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) - - if "+cuda" in spec: - entries.append(cmake_cache_option("ENABLE_CUDA", True)) - if "+separable_compilation" in spec: - entries.append(cmake_cache_option("CMAKE_CUDA_SEPARABLE_COMPILATION", True)) - entries.append(cmake_cache_option("CUDA_SEPARABLE_COMPILATION", True)) - - if not spec.satisfies("cuda_arch=none"): - cuda_arch = spec.variants["cuda_arch"].value - entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) - entries.append( - cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) - ) - flag = "-arch sm_{0}".format(cuda_arch[0]) - entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) - else: - entries.append(cmake_cache_option("ENABLE_CUDA", False)) - - if "+rocm" in spec: - entries.append(cmake_cache_option("ENABLE_HIP", True)) - entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) - archs = self.spec.variants["amdgpu_target"].value - if archs != "none": - arch_str = ",".join(archs) - entries.append( - cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) - ) - else: - entries.append(cmake_cache_option("ENABLE_HIP", False)) - - return entries - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - option_prefix = "CHAI_" if spec.satisfies("@2022.03.0:") else "" - - entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) - if "+raja" in spec: - entries.append(cmake_cache_option("{}ENABLE_RAJA_PLUGIN".format(option_prefix), True)) - entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) - entries.append( - cmake_cache_option("{}ENABLE_PICK".format(option_prefix), "+enable_pick" in spec) - ) - if spec.satisfies("@2024.02:"): - entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.lib64.cmake.umpire)) - else: - entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix.share.umpire.cmake)) - entries.append(cmake_cache_option("ENABLE_TESTS", "+tests" in spec)) - entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "+benchmarks" in spec)) - entries.append( - cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) - ) - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) - - return entries - - def cmake_args(self): - options = [] - return options diff --git a/repo/kripke/.package.py.swp b/repo/kripke/.package.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..07395690e74d7b3a90206ca2702bc94909c59f72 GIT binary patch literal 16384 zcmeHOOKc=Z8EyjE4QvA0F(`^9~*X&00|I6;t)=VLoOj477j=pKtw{EP)?u)BoGopgbNqM1uhW2>c`A@JRZCf zhX{2`-?XdiufP8K>aXhhtJ;eEskL=dm1F_evx4AUl=WZC-4kArM4?CT1}x7rreAjh z>iYMJ)lkP2(OlrUxleE9bCKelYP;0T7IF%+)Sg4#y=pJt^Dqzp`D;2EJ<$cMvpMtp&M^wQ2$=`=HCAY~wBAY~wBAY~wBAY~wBAZ6hH zivizxLihrj^hDgOetdu5*!}%+dN#gWWBGA1y`>DK45SRC45SRC45SRC45SRC45SRC z45SRC47>*!pqe0@gI&Mj0YA?FllcE9AHmNcnG`( z8~~pH%D@%iIpEz73BsR%?*gv_uK`~I?f~n+CE#h`0`T?+1>y6+3?KsUJ|zf`fH#2G zfdilgTn941JLd(#2RvXKcoujBvc3gqfC?-C=Yc;wDF{CVz6NxG+kgc8`vZdTE8q>_ z9Dxv{qfM(`BBMr^&-SmUbag)n2}2Qq4p8r|i!PJPxB z$t>}hsY~4KnAZvdN{Ryy@kaTHD>N_D-j{ zGFgXW`qLFO8|#~si>efiPj2|l0fnJkutT4_#juC@Q&J~4huy)s#FGK9p#qKRMD5c( z*5w54v+81$iaP(K?A$TgD0Q0b12xd-P;Zzz zO?K{>Y?L}p_Cf5k1B>JIY4USp@=5wiMC)iHxl5jn4la2%GD4llnimms(%ewzG0D@2 z2z8D%E1~5Sy<^g2BoHz}gD!Fpz543@{iOGF@iwnAd=%*)FA(QGV23DbGH)&%p>Y@e z*W!^Qa`giRXRex86qjVKYIw{$gAUzx$v&S!F`4dKOzZA5*E4L(i$}?f%kBo4KfKIK zDxUf-HLNbDV4gU(X{bka(Y9FE)?sGf+pUY6hPp>6;UXl~MGg$VM;sT-yHP!TX8GhG z$EP)Nne2o!R6M=U9Ipl=+B7^rGs+w})##{8#+$p?RXX6rU5jCBwOo*NP9|<&zQjJy1jCe3FSS3DpXNE{9|YUPwjt6T`y`(KGmR03L^# zna`~Aq_|ABEX>C~b9qV6H&lYJ%W*TApqkY4NTeXsT*nG_Gy1lCo;0zV93(pJWy@MG4$AAtnU=6qk`~|W7cYrQX1|A`P|1z)-tOCyi9|QiyV|d^ISOT5|eujA6 z0zL_x10EtK|2FUq;B!D1xDLDm%mROeO+N?T1h{>l1=1^JAY~wBAY~wBAY~wB;62R% z4^jAKV1tL6X6BBKO&*8(y>P=+3b#1A8g_wH-*&My6Qh53l-MB_@Yo0nH?Aecu!d22 ze-H9bW>VG+){ajYAM%SonQ#!X zv)rh+S6300gi`on*il&=q>`t@{NxG0&ytADc*{Rf1{>d!VX0=I@z`K^CcyValAZBH zV6tKyEi~5~*Ktw;1G}rO#>-pvY?d85K7RPFIn%Nx?JF83*|k{@Kl}tj7`2W6J=*B0 znaAsEbeh-JwpW_lJDruS`m6GNQNr$ApZXcTe~VprwRcm#b>*b#qxEr>+<30+adZr- zKtRWNmEv704`(V#?O<`AngKi3CY&E}*9yyc2t7OmK?pnth3XbIHAOfJh#6d?7Kc3r zS3|^_5l861*52CgHrv>#YOQp56h7Xg=vVddaEy;p&EO(A`3fEG)IL)ypB08Z_^A}vAh7WS2(<*O~9Vv@Mu*tym0eRNZLNx(Z|e=Xh3CQKTdWEerggr zg*Qe4uW3vl*90`11E;2aZPy91(F}0RQN(#s;bjX`eY2FpD0IRM%d2+m*-b_gvy(G7GIj=?l*4yuN)ntWx|0hp8>:-rdynamic>> diff --git a/repo/umpire/missing_header_for_numeric_limits.patch b/repo/umpire/missing_header_for_numeric_limits.patch deleted file mode 100644 index 6e89e3707..000000000 --- a/repo/umpire/missing_header_for_numeric_limits.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/umpire/util/allocation_statistics.cpp b/src/umpire/util/allocation_statistics.cpp -index 597ae61f..0c3f7865 100644 ---- a/src/umpire/util/allocation_statistics.cpp -+++ b/src/umpire/util/allocation_statistics.cpp -@@ -8,6 +8,7 @@ - #include "umpire/util/allocation_statistics.hpp" - - #include -+#include - - namespace umpire { - namespace util { diff --git a/repo/umpire/package.py b/repo/umpire/package.py deleted file mode 100644 index 7c23151c4..000000000 --- a/repo/umpire/package.py +++ /dev/null @@ -1,376 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import socket - -import llnl.util.tty as tty - -from spack.package import * -from spack.pkg.builtin.camp import hip_repair_cache - - -class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): - """An application-focused API for memory management on NUMA & GPU - architectures""" - - homepage = "https://github.com/LLNL/Umpire" - git = "https://github.com/LLNL/Umpire.git" - tags = ["radiuss", "e4s"] - - maintainers("davidbeckingsale") - - license("MIT") - - version("develop", branch="develop", submodules=False) - version("main", branch="main", submodules=False) - version( - "2024.02.1", - tag="v2024.02.1", - commit="3058d562fc707650e904f9321b1ee9bcebad3ae2", - submodules=True, - ) - version( - "2022.10.0", - tag="v2022.10.0", - commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a", - submodules=False, - ) - version( - "2022.03.1", - tag="v2022.03.1", - commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1", - submodules=False, - ) - version( - "2022.03.0", - tag="v2022.03.0", - commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b", - submodules=False, - ) - version( - "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True - ) - version( - "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True - ) - version( - "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True - ) - version( - "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True - ) - version( - "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True - ) - version( - "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True - ) - version( - "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True - ) - version( - "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True - ) - version( - "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True - ) - version( - "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True - ) - version( - "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True - ) - version( - "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True - ) - version( - "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True - ) - version( - "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True - ) - version( - "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True - ) - version( - "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True - ) - version( - "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True - ) - version( - "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True - ) - version( - "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True - ) - version( - "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True - ) - version( - "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True - ) - version( - "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True - ) - version( - "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True - ) - version( - "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True - ) - version( - "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True - ) - version( - "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True - ) - version( - "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True - ) - - patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:") - patch("camp_target_umpire_3.0.0.patch", when="@3.0.0") - patch("cmake_version_check.patch", when="@4.1") - patch("missing_header_for_numeric_limits.patch", when="@4.1:5.0.1") - - # export targets when building pre-6.0.0 release with BLT 0.4.0+ - patch( - "https://github.com/LLNL/Umpire/commit/5773ce9af88952c8d23f9bcdcb2e503ceda40763.patch?full_index=1", - sha256="f3b21335ce5cf9c0fecc852a94dfec90fb5703032ac97f9fee104af9408d8899", - when="@:5.0.1 ^blt@0.4:", - ) - - # https://github.com/LLNL/Umpire/pull/805 - patch( - "https://github.com/LLNL/Umpire/pull/805/commits/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1", - sha256="7ed5d2c315a3b31e339f664f6108e32d7cb4cb8e9f22e5c78a65ba02625ccc09", - when="@2022.10.0", - ) - - # https://github.com/LLNL/Umpire/pull/816 - patch( - "https://github.com/LLNL/Umpire/pull/816/commits/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1", - sha256="0f43cad7cdaec3c225ab6414ab9f81bd405a1157abf5a508e515bcb6ca53326d", - when="@2022.10.0", - ) - - # https://github.com/LLNL/Umpire/pull/853 - patch( - "https://github.com/LLNL/Umpire/commit/4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256.patch?full_index=1", - sha256="c9ddae1f4212cef72e1050b6ac482ce5b795dad4977d2462cff2e884b8d7aff5", - when="@2022.10:2023.06", - ) - - variant("fortran", default=False, description="Build C/Fortran API") - variant("c", default=True, description="Build C API") - variant("numa", default=False, description="Enable NUMA support") - variant("shared", default=True, description="Enable Shared libs") - variant("openmp", default=False, description="Build with OpenMP support") - variant("deviceconst", default=False, description="Enables support for constant device memory") - variant("examples", default=True, description="Build Umpire Examples") - variant( - "tests", - default="none", - values=("none", "basic", "benchmarks"), - multi=False, - description="Tests to run", - ) - variant("device_alloc", default=True, description="Build Umpire Device Allocator") - - depends_on("cmake@3.8:", type="build") - depends_on("cmake@3.9:", when="+cuda", type="build") - depends_on("cmake@3.14:", when="@2022.03.0:", type="build") - - depends_on("blt", type="build") - depends_on("blt@0.6.2", type="build", when="@2024.02:") - depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") - depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2022.03.1") - depends_on("blt@0.4.1", type="build", when="@6.0.0") - depends_on("blt@0.4.0:", type="build", when="@4.1.3:5.0.1") - depends_on("blt@0.3.6:", type="build", when="@:4.1.2") - conflicts("^blt@:0.3.6", when="+rocm") - - depends_on("camp", when="@5.0.0:") - depends_on("camp@0.2.2:0.2.3", when="@6.0.0") - depends_on("camp@0.1.0", when="@5.0.0:5.0.1") - depends_on("camp@2022.03.2:", when="@2022.03.0:2024.01") - depends_on("camp@2024.02:", when="@2024.02:") - depends_on("camp@main", when="@main") - depends_on("camp@main", when="@develop") - depends_on("camp+openmp", when="+openmp") - depends_on("camp~cuda", when="~cuda") - depends_on("camp~rocm", when="~rocm") - - with when("@5.0.0:"): - with when("+cuda"): - depends_on("camp+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("camp+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - - with when("+rocm"): - depends_on("camp+rocm") - for arch_ in ROCmPackage.amdgpu_targets: - depends_on( - "camp+rocm amdgpu_target={0}".format(arch_), - when="amdgpu_target={0}".format(arch_), - ) - - conflicts("+numa", when="@:0.3.2") - conflicts("~c", when="+fortran", msg="Fortran API requires C API") - conflicts("+device_alloc", when="@:2022.03.0") - - # device allocator exports device code, which requires static libs - # currently only available for cuda. - conflicts("+shared", when="+cuda") - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip("1234567890") - return "{0}-{1}-{2}@{3}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version, - ) - - def initconfig_compiler_entries(self): - spec = self.spec - entries = super().initconfig_compiler_entries() - - if "+rocm" in spec: - entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - if "+fortran" in spec and self.compiler.fc is not None: - entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) - else: - entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) - - entries.append(cmake_cache_option("{}ENABLE_C".format(option_prefix), "+c" in spec)) - - return entries - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super().initconfig_hardware_entries() - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - if "+cuda" in spec: - entries.append(cmake_cache_option("ENABLE_CUDA", True)) - - if not spec.satisfies("cuda_arch=none"): - cuda_arch = spec.variants["cuda_arch"].value - entries.append(cmake_cache_string("CUDA_ARCH", "sm_{0}".format(cuda_arch[0]))) - entries.append( - cmake_cache_string("CMAKE_CUDA_ARCHITECTURES", "{0}".format(cuda_arch[0])) - ) - flag = "-arch sm_{0}".format(cuda_arch[0]) - entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", "{0}".format(flag))) - - entries.append( - cmake_cache_option( - "{}ENABLE_DEVICE_CONST".format(option_prefix), spec.satisfies("+deviceconst") - ) - ) - else: - entries.append(cmake_cache_option("ENABLE_CUDA", False)) - - if "+rocm" in spec: - entries.append(cmake_cache_option("ENABLE_HIP", True)) - entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix))) - hip_repair_cache(entries, spec) - archs = self.spec.variants["amdgpu_target"].value - if archs != "none": - arch_str = ",".join(archs) - entries.append( - cmake_cache_string("HIP_HIPCC_FLAGS", "--amdgpu-target={0}".format(arch_str)) - ) - entries.append( - cmake_cache_string("CMAKE_HIP_ARCHITECTURES", "{0}".format(arch_str)) - ) - else: - entries.append(cmake_cache_option("ENABLE_HIP", False)) - - return entries - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - # TPL locations - entries.append("#------------------{0}".format("-" * 60)) - entries.append("# TPLs") - entries.append("#------------------{0}\n".format("-" * 60)) - - entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) - if spec.satisfies("@5.0.0:"): - entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) - entries.append(cmake_cache_option("{}ENABLE_NUMA".format(option_prefix), "+numa" in spec)) - entries.append(cmake_cache_option("ENABLE_OPENMP", "+openmp" in spec)) - entries.append(cmake_cache_option("ENABLE_BENCHMARKS", "tests=benchmarks" in spec)) - entries.append( - cmake_cache_option("{}ENABLE_EXAMPLES".format(option_prefix), "+examples" in spec) - ) - entries.append(cmake_cache_option("{}ENABLE_DOCS".format(option_prefix), False)) - entries.append( - cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", "+device_alloc" in spec) - ) - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", "+shared" in spec)) - entries.append(cmake_cache_option("ENABLE_TESTS", "tests=none" not in spec)) - - return entries - - def cmake_args(self): - options = [] - return options - - def test(self): - """Perform stand-alone checks on the installed package.""" - if self.spec.satisfies("@:1") or not os.path.isdir(self.prefix.bin): - tty.info("Skipping: checks not installed in bin for v{0}".format(self.version)) - return - - # Run a subset of examples PROVIDED installed - # tutorials with readily checkable outputs. - checks = { - "malloc": ["99 should be 99"], - "recipe_dynamic_pool_heuristic": ["in the pool", "releas"], - "recipe_no_introspection": ["has allocated", "used"], - "strategy_example": ["Available allocators", "HOST"], - "tut_copy": ["Copied source data"], - "tut_introspection": ["Allocator used is HOST", "size of the allocation"], - "tut_memset": ["Set data from HOST"], - "tut_move": ["Moved source data", "HOST"], - "tut_reallocate": ["Reallocated data"], - "vector_allocator": [""], - } - - for exe in checks: - expected = checks[exe] - reason = "test: checking output from {0}".format(exe) - self.run_test( - exe, - [], - expected, - 0, - installed=False, - purpose=reason, - skip_missing=True, - work_dir=self.prefix.bin, - ) diff --git a/repo/umpire/std-filesystem-pr784.patch b/repo/umpire/std-filesystem-pr784.patch deleted file mode 100644 index 677144214..000000000 --- a/repo/umpire/std-filesystem-pr784.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff -ruN spack-src/cmake/SetupCompilerFlags.cmake spack-src-patched/cmake/SetupCompilerFlags.cmake ---- spack-src/cmake/SetupCompilerFlags.cmake 2022-10-12 08:05:03.538390165 -0700 -+++ spack-src-patched/cmake/SetupCompilerFlags.cmake 2022-10-12 09:47:56.317645003 -0700 -@@ -11,20 +11,20 @@ - - message(STATUS "Checking for std::filesystem") - --include(CheckCXXSourceCompiles) --check_cxx_source_compiles( -- "#include -- #include -- -- int main(int, char**) -- { -+blt_check_code_compiles(CODE_COMPILES UMPIRE_ENABLE_FILESYSTEM -+ SOURCE_STRING -+[=[ -+#include -+#include - -+int main(int, char**) -+{ - auto path = std::filesystem::path(\".\"); - (void)(path); - - return 0; -- }" -- UMPIRE_ENABLE_FILESYSTEM) -+} -+]=]) - - if (UMPIRE_ENABLE_FILESYSTEM) - message(STATUS "std::filesystem found") From 394a44e383fcc3def5297caea998d0dcf98bd9d0 Mon Sep 17 00:00:00 2001 From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:06:34 -0700 Subject: [PATCH 04/14] Delete repo/kripke/application.py --- repo/kripke/application.py | 118 ------------------------------------- 1 file changed, 118 deletions(-) delete mode 100644 repo/kripke/application.py diff --git a/repo/kripke/application.py b/repo/kripke/application.py deleted file mode 100644 index 61a79f7f6..000000000 --- a/repo/kripke/application.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from ramble.appkit import * - -import sys - -class Kripke(SpackApplication): - """Kripke benchmark uses RAJA Portability Layer""" - name = "Kripke" - - tags = ['asc','transport','deterministic','structured-grid', - 'large-scale','multi-node','single-node','c++','raja', - 'simd','vectorization','register-pressure','high-fp','atomics','high-branching', - 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', - 'mpi','network-latency-bound','network-collectives'] - - executable('kripke', 'kripke.exe' + - ' --groups {ngroups}' + - ' --legendre {lorder}' + - ' --quad {nquad}' + - ' --zones {nzx},{nzy},{nzz}' + - ' --sigt {sigt0},{sigt1},{sigt2}' + - ' --sigs {sigs0},{sigs1},{sigs2}' + - ' --arch {arch}' + - ' --layout {layout}' + -# ' --pdist {lout}' + - ' --procs {npx},{npy},{npz}' + - ' --dset {ds}' + - ' --gset {gs}' + - ' --zset {nzsetx},{nzsety},{nzsetz}' + - ' --niter {niter}' + - ' --pmethod {method}', use_mpi=True) - - workload('kripke', executables=['kripke']) - - workload_variable('ngroups', default='32', - description='Number of energy groups. (Default: --groups 32)', - workloads=['kripke']) - workload_variable('lorder', default='4', - description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', - workloads=['kripke']) - workload_variable('nquad', default='96', - description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', - workloads=['kripke']) - workload_variable('nzx', default='16', - description='Number of zones in x. (Default: 16)', - workloads=['kripke']) - workload_variable('nzx', default='16', - description='Number of zones in y. (Default: 16)', - workloads=['kripke']) - workload_variable('nzz', default='16', - description='Number of zones in z. (Default: 16)', - workloads=['kripke']) - workload_variable('sigt0', default='0.1', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('sigt1', default='0.0001', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('sigt2', default='0.1', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('sigs0', default='0.05', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('sigs1', default='0.00005', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('sigs2', default='0.05', - description='Total material cross-sections', - workloads=['kripke']) - workload_variable('arch', default='OpenMP', - description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', - workloads=['kripke']) - workload_variable('layout', default='DGZ', - description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', - workloads=['kripke']) -# workload_variable('lout', default='0', -# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', -# workloads=['kripke']) - workload_variable('npx', default='1', - description='Number of MPI ranks in x dimension', - workloads=['kripke']) - workload_variable('npy', default='1', - description='Number of MPI ranks in y dimension', - workloads=['kripke']) - workload_variable('npz', default='1', - description='Number of MPI ranks in z dimension', - workloads=['kripke']) - workload_variable('ds', default='8', - description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', - workloads=['kripke']) - workload_variable('gs', default='1', - description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', - workloads=['kripke']) - workload_variable('nzsetx', default='1', - description='Number of zone-sets in x', - workloads=['kripke']) - workload_variable('nzsety', default='1', - description='Number of zone-sets in y', - workloads=['kripke']) - workload_variable('nzsetz', default='1', - description='Number of zone-sets in z', - workloads=['kripke']) - workload_variable('niter', default='10', - description='Number of solver iterations to run. (Default: --niter 10)', - workloads=['kripke']) - workload_variable('method', default='sweep', - description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', - workloads=['kripke']) - - #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') - - #TODO: Fix the FOM success_criteria(...) - #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') From 64b94b9fc2a77eb949cf3ea462310519b47e813f Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Mon, 29 Apr 2024 15:09:14 -0700 Subject: [PATCH 05/14] Add kripke application.py again --- repo/kripke/application.py | 118 +++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 repo/kripke/application.py diff --git a/repo/kripke/application.py b/repo/kripke/application.py new file mode 100644 index 000000000..61a79f7f6 --- /dev/null +++ b/repo/kripke/application.py @@ -0,0 +1,118 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from ramble.appkit import * + +import sys + +class Kripke(SpackApplication): + """Kripke benchmark uses RAJA Portability Layer""" + name = "Kripke" + + tags = ['asc','transport','deterministic','structured-grid', + 'large-scale','multi-node','single-node','c++','raja', + 'simd','vectorization','register-pressure','high-fp','atomics','high-branching', + 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', + 'mpi','network-latency-bound','network-collectives'] + + executable('kripke', 'kripke.exe' + + ' --groups {ngroups}' + + ' --legendre {lorder}' + + ' --quad {nquad}' + + ' --zones {nzx},{nzy},{nzz}' + + ' --sigt {sigt0},{sigt1},{sigt2}' + + ' --sigs {sigs0},{sigs1},{sigs2}' + + ' --arch {arch}' + + ' --layout {layout}' + +# ' --pdist {lout}' + + ' --procs {npx},{npy},{npz}' + + ' --dset {ds}' + + ' --gset {gs}' + + ' --zset {nzsetx},{nzsety},{nzsetz}' + + ' --niter {niter}' + + ' --pmethod {method}', use_mpi=True) + + workload('kripke', executables=['kripke']) + + workload_variable('ngroups', default='32', + description='Number of energy groups. (Default: --groups 32)', + workloads=['kripke']) + workload_variable('lorder', default='4', + description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', + workloads=['kripke']) + workload_variable('nquad', default='96', + description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in x. (Default: 16)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in y. (Default: 16)', + workloads=['kripke']) + workload_variable('nzz', default='16', + description='Number of zones in z. (Default: 16)', + workloads=['kripke']) + workload_variable('sigt0', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt1', default='0.0001', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt2', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs0', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs1', default='0.00005', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs2', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('arch', default='OpenMP', + description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', + workloads=['kripke']) + workload_variable('layout', default='DGZ', + description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', + workloads=['kripke']) +# workload_variable('lout', default='0', +# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', +# workloads=['kripke']) + workload_variable('npx', default='1', + description='Number of MPI ranks in x dimension', + workloads=['kripke']) + workload_variable('npy', default='1', + description='Number of MPI ranks in y dimension', + workloads=['kripke']) + workload_variable('npz', default='1', + description='Number of MPI ranks in z dimension', + workloads=['kripke']) + workload_variable('ds', default='8', + description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', + workloads=['kripke']) + workload_variable('gs', default='1', + description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', + workloads=['kripke']) + workload_variable('nzsetx', default='1', + description='Number of zone-sets in x', + workloads=['kripke']) + workload_variable('nzsety', default='1', + description='Number of zone-sets in y', + workloads=['kripke']) + workload_variable('nzsetz', default='1', + description='Number of zone-sets in z', + workloads=['kripke']) + workload_variable('niter', default='10', + description='Number of solver iterations to run. (Default: --niter 10)', + workloads=['kripke']) + workload_variable('method', default='sweep', + description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', + workloads=['kripke']) + + #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + + #TODO: Fix the FOM success_criteria(...) + #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') From 93184efb3ce9de21d7d25562d0fe0b5d47ffa682 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Mon, 29 Apr 2024 15:09:48 -0700 Subject: [PATCH 06/14] Remove redundant file --- repo/kripke/.package.py.swp | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 repo/kripke/.package.py.swp diff --git a/repo/kripke/.package.py.swp b/repo/kripke/.package.py.swp deleted file mode 100644 index 07395690e74d7b3a90206ca2702bc94909c59f72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOOKc=Z8EyjE4QvA0F(`^9~*X&00|I6;t)=VLoOj477j=pKtw{EP)?u)BoGopgbNqM1uhW2>c`A@JRZCf zhX{2`-?XdiufP8K>aXhhtJ;eEskL=dm1F_evx4AUl=WZC-4kArM4?CT1}x7rreAjh z>iYMJ)lkP2(OlrUxleE9bCKelYP;0T7IF%+)Sg4#y=pJt^Dqzp`D;2EJ<$cMvpMtp&M^wQ2$=`=HCAY~wBAY~wBAY~wBAY~wBAZ6hH zivizxLihrj^hDgOetdu5*!}%+dN#gWWBGA1y`>DK45SRC45SRC45SRC45SRC45SRC z45SRC47>*!pqe0@gI&Mj0YA?FllcE9AHmNcnG`( z8~~pH%D@%iIpEz73BsR%?*gv_uK`~I?f~n+CE#h`0`T?+1>y6+3?KsUJ|zf`fH#2G zfdilgTn941JLd(#2RvXKcoujBvc3gqfC?-C=Yc;wDF{CVz6NxG+kgc8`vZdTE8q>_ z9Dxv{qfM(`BBMr^&-SmUbag)n2}2Qq4p8r|i!PJPxB z$t>}hsY~4KnAZvdN{Ryy@kaTHD>N_D-j{ zGFgXW`qLFO8|#~si>efiPj2|l0fnJkutT4_#juC@Q&J~4huy)s#FGK9p#qKRMD5c( z*5w54v+81$iaP(K?A$TgD0Q0b12xd-P;Zzz zO?K{>Y?L}p_Cf5k1B>JIY4USp@=5wiMC)iHxl5jn4la2%GD4llnimms(%ewzG0D@2 z2z8D%E1~5Sy<^g2BoHz}gD!Fpz543@{iOGF@iwnAd=%*)FA(QGV23DbGH)&%p>Y@e z*W!^Qa`giRXRex86qjVKYIw{$gAUzx$v&S!F`4dKOzZA5*E4L(i$}?f%kBo4KfKIK zDxUf-HLNbDV4gU(X{bka(Y9FE)?sGf+pUY6hPp>6;UXl~MGg$VM;sT-yHP!TX8GhG z$EP)Nne2o!R6M=U9Ipl=+B7^rGs+w})##{8#+$p?RXX6rU5jCBwOo*NP9|<&zQjJy1jCe3FSS3DpXNE{9|YUPwjt6T`y`(KGmR03L^# zna`~Aq_|ABEX>C~b9qV6H&lYJ%W*TApqkY4NTeXsT*nG_Gy1lCo;0zV93(pJWy@MG4$AAtnU=6qk`~|W7cYrQX1|A`P|1z)-tOCyi9|QiyV|d^ISOT5|eujA6 z0zL_x10EtK|2FUq;B!D1xDLDm%mROeO+N?T1h{>l1=1^JAY~wBAY~wBAY~wB;62R% z4^jAKV1tL6X6BBKO&*8(y>P=+3b#1A8g_wH-*&My6Qh53l-MB_@Yo0nH?Aecu!d22 ze-H9bW>VG+){ajYAM%SonQ#!X zv)rh+S6300gi`on*il&=q>`t@{NxG0&ytADc*{Rf1{>d!VX0=I@z`K^CcyValAZBH zV6tKyEi~5~*Ktw;1G}rO#>-pvY?d85K7RPFIn%Nx?JF83*|k{@Kl}tj7`2W6J=*B0 znaAsEbeh-JwpW_lJDruS`m6GNQNr$ApZXcTe~VprwRcm#b>*b#qxEr>+<30+adZr- zKtRWNmEv704`(V#?O<`AngKi3CY&E}*9yyc2t7OmK?pnth3XbIHAOfJh#6d?7Kc3r zS3|^_5l861*52CgHrv>#YOQp56h7Xg=vVddaEy;p&EO(A`3fEG)IL)ypB08Z_^A}vAh7WS2(<*O~9Vv@Mu*tym0eRNZLNx(Z|e=Xh3CQKTdWEerggr zg*Qe4uW3vl*90`11E;2aZPy91(F}0RQN(#s;bjX`eY2FpD0IRM%d2+m*-b_gvy(G7GIj=?l*4yuN)ntWx|0hp8 Date: Tue, 30 Apr 2024 10:58:34 -0700 Subject: [PATCH 07/14] CUDA implementation of kripke --- .../auxiliary_software_files/packages.yaml | 9 ++++++++- experiments/kripke/cuda/ramble.yaml | 4 ++-- repo/kripke/package.py | 20 ++++++++++--------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index a13d6f3b3..42cffef81 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -18,7 +18,9 @@ packages: externals: - spec: cmake@3.23.1 prefix: /usr/tce/packages/cmake/cmake-3.23.1 - buildable: false + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 + buildable: false gmake: externals: - spec: gmake@4.2.1 @@ -48,6 +50,11 @@ packages: prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/ - spec: mvapich2@2.3.7-clang-14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6 + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: buildable: false externals: diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml index 25cc12855..b4771bdba 100644 --- a/experiments/kripke/cuda/ramble.yaml +++ b/experiments/kripke/cuda/ramble.yaml @@ -23,8 +23,8 @@ ramble: variables: n_ranks: '{processes_per_node} * {n_nodes}' experiment_setup: '' - processes_per_node: ['4'] - n_nodes: '1' + processes_per_node: '2' + n_nodes: '2' arch: 'CUDA' experiments: kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 68e84539b..0aacc2b55 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -24,8 +24,6 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): version( "1.2.7.0", submodules=False, commit="db920c1f5e1dcbb9e949d120e7d86efcdb777635" ) - version( - "updateRCU2024.02", branch="task/chen59/updateRCU2024.02", submodules=False) version( "1.2.4", submodules=False, tag="v1.2.4", commit="d85c6bc462f17a2382b11ba363059febc487f771" ) @@ -58,20 +56,19 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): depends_on("mpi", when="+mpi") depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") - - depends_on("blt@0.6.2", type="build") conflicts("^blt@:0.3.6", when="+rocm") aligned_versions = ["2024.02"] #constrain_chai = list(f"^chai@{v}" for v in aligned_versions) #requires(*constrain_chai, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") - for v in aligned_versions: depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") depends_on(f"umpire@{v}~examples", when=f"^chai@{v}") depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}") depends_on(f"camp@{v}", when=f"^chai@{v}") + depends_on("blt@0.6.2:", type="build", when=f"@1.2.7:") + depends_on("chai+openmp", when="+openmp") depends_on("chai~openmp", when="~openmp") depends_on("chai+cuda", when="+cuda") @@ -86,6 +83,11 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): depends_on("umpire+rocm", when="+rocm") depends_on("umpire~rocm", when="~rocm") + def setup_build_environment(self, env): + spec = self.spec + if "+cuda" in spec: + env.set("CUDAHOSTCXX", self.spec["mpi"].mpicxx) + def cmake_args(self): spec = self.spec args = [] @@ -130,10 +132,10 @@ def cmake_args(self): cuda_arch = spec.variants["cuda_arch"].value args.append("-DCUDA_ARCH={0}".format(cuda_arch[0])) args.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) - args.append( - "-DCMAKE_CUDA_FLAGS=--expt-extended-lambda -I%s -I=%s" - % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include) - ) + args.append( + "-DCMAKE_CUDA_FLAGS=--extended-lambda -I%s -I=%s" + % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include) + ) else: args.append("-DENABLE_CUDA=OFF") From 852a44ab0355c3b919f6b5bfef638b0e7e9bf747 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 30 Apr 2024 11:42:58 -0700 Subject: [PATCH 08/14] Upgrade cmake externals package --- .../auxiliary_software_files/packages.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml index 0cb4fc3b5..7bb349eec 100644 --- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -18,7 +18,9 @@ packages: externals: - spec: cmake@3.23.1 prefix: /usr/tce/packages/cmake/cmake-3.23.1 - buildable: false + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 + buildable: false gmake: externals: - spec: gmake@4.2.1 From a5ddc089b6cad33fda1846adac05dd8c47781326 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 30 Apr 2024 12:11:12 -0700 Subject: [PATCH 09/14] Update cmake external package config for LLNL-Tioga-HPECray-zen3-MI250X-Slingshot --- .../auxiliary_software_files/packages.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml index af604fd29..a9d63480b 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml @@ -41,6 +41,7 @@ packages: prefix: /usr - spec: cmake@3.23.1 prefix: /usr/tce + buildable: false pkgconf: externals: - spec: pkgconf@1.4.2 From e8f865cff5d52fc4c41a055abb2a1b95e4fda2fa Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 30 Apr 2024 14:05:09 -0700 Subject: [PATCH 10/14] Update lassen compiler config --- .../auxiliary_software_files/compilers.yaml | 20 +++++++++++++++++-- .../auxiliary_software_files/packages.yaml | 11 ++++++++++ .../spack.yaml | 4 ++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml index d27a632c1..4e4c457a1 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml @@ -4,6 +4,22 @@ # SPDX-License-Identifier: Apache-2.0 compilers: +- compiler: + spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1 + paths: + cc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlc + cxx: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlC + f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + flags: + cflags: -g -O2 + cxxflags: -g -O2 -std=c++14 + fflags: -g -O2 + operating_system: rhel7 + target: ppc64le + modules: [cuda/11.8.0, xl/2023.06.28-cuda-11.8.0-gcc-11.2.1] + environment: {} + extra_rpaths: [] - compiler: spec: xl@16.1.1-2022.08.19-cuda-11.7.0 paths: @@ -29,7 +45,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf flags: cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le @@ -45,7 +61,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf flags: # TODO: Fix spack concretization bug cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml index 61c85f62d..886aabfd1 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml @@ -13,6 +13,8 @@ packages: externals: - spec: cmake@3.23.1 prefix: /usr/tce/packages/cmake/cmake-3.23.1 + - spec: cmake@3.29.2 + prefix: /usr/tce/packages/cmake/cmake-3.29.2 buildable: false gmake: externals: @@ -41,6 +43,11 @@ packages: - spec: cusparse@11.8.0 prefix: /usr/tce/packages/cuda/cuda-11.8.0 buildable: false + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: externals: - spec: cuda@10.1.243 @@ -72,6 +79,10 @@ packages: buildable: false mpi: externals: + - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1 + prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1 + extra_attributes: + ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm" - spec: spectrum-mpi@2022.08.19-cuda-11.8.0 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0 extra_attributes: diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml index ed1edcb60..7b2f4257b 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml @@ -6,9 +6,9 @@ spack: packages: default-compiler: - spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version} + spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 default-mpi: - spack_spec: spectrum-mpi@2022.08.19 + spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 compiler-gcc: spack_spec: gcc@8.3.1 compiler-clang: From d635bfed1f31d920dd540cb5db4567e7bd248170 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Apr 2024 17:37:28 -0500 Subject: [PATCH 11/14] Removing deprecated cmake from machine config --- .../auxiliary_software_files/packages.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml index 7bb349eec..7aa44dea9 100644 --- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -16,8 +16,6 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 - spec: cmake@3.26.3 prefix: /usr/tce/packages/cmake/cmake-3.26.3 buildable: false From 2103c27de33841fac0e3abea7f5949b5431a0ab2 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Apr 2024 17:38:20 -0500 Subject: [PATCH 12/14] Removing deprecated cmake from machine config --- .../auxiliary_software_files/packages.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index 42cffef81..129876c20 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -16,8 +16,6 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 - spec: cmake@3.26.3 prefix: /usr/tce/packages/cmake/cmake-3.26.3 buildable: false From 4e77d3e63200e98090d488f57de20631303ba152 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Apr 2024 17:39:07 -0500 Subject: [PATCH 13/14] Removing deprecated cmake from machine config --- .../auxiliary_software_files/packages.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml index 886aabfd1..5d26658c1 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml @@ -11,8 +11,6 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 - spec: cmake@3.29.2 prefix: /usr/tce/packages/cmake/cmake-3.29.2 buildable: false From b40546e03be3d998fcca9bee34b594ffb698dce3 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Apr 2024 17:42:31 -0500 Subject: [PATCH 14/14] minor --- repo/kripke/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 0aacc2b55..7f1269550 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -59,8 +59,6 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): conflicts("^blt@:0.3.6", when="+rocm") aligned_versions = ["2024.02"] - #constrain_chai = list(f"^chai@{v}" for v in aligned_versions) - #requires(*constrain_chai, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") for v in aligned_versions: depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") depends_on(f"umpire@{v}~examples", when=f"^chai@{v}")