OpenMP implementation of Kripke

experiments/kripke/openmp/execute_experiment.tpl

@@ -1,5 +1,4 @@
 #!/bin/bash
-#
 # Copyright 2023 Lawrence Livermore National Security, LLC and other
 # Benchpark Project Developers. See the top-level COPYRIGHT file for details.
 #
@@ -11,8 +10,4 @@
 
 cd {experiment_run_dir}
 
-{spack_setup}
-
-{experiment_setup}
-
 {command}

experiments/kripke/openmp/ramble.yaml

@@ -7,6 +7,7 @@ ramble:
   include:
   - ./configs/spack.yaml
   - ./configs/variables.yaml
+  - ./configs/modifier.yaml
 
   config:
     deprecated: true
@@ -17,41 +18,40 @@ ramble:
   applications:
     kripke:
       workloads:
-        problem1:
+        kripke:
           env_vars:
             set:
               OMP_NUM_THREADS: '{omp_num_threads}'
           variables:
             n_ranks: '{processes_per_node} * {n_nodes}'
-            p: 2
-            px: '{p}'
-            py: '{p}'
-            pz: '{p}'
-            n: ['55', '110']
-            nx: '{n}'
-            ny: '{n}'
-            nz: '{n}'
             experiment_setup: ''
-            processes_per_node: ['8', '4']
-            n_nodes: ['1', '2']
-            threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks
-            omp_num_threads: '{threads_per_node_core} * {n_nodes}'
+            processes_per_node: ['1', '16']
+            n_nodes: '1'
+            omp_num_threads: ['32', '2']
+            arch: 'OpenMP'
           experiments:
-            kripke_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
+            kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
               variables:
-                env_name: kripke
-              matrices:
-                - size_threads:
-                  - n # TODO: Filter matrix
-                  - threads_per_node_core # TODO: Filter matrix
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '32'
+                nzz: '32'
+                npx: ['1', '4']
+                npy: ['1', '2']
+                npz: ['1', '2']
   spack:
     concretized: true
     packages:
-      kripke-omp:
-        spack_spec: kripke@develop +mpi+openmp
+      kripke:
+        spack_spec: kripke@develop +mpi+openmp{modifier_spack_variant} ^[email protected]
         compiler: default-compiler
     environments:
-      kripke-omp:
+      kripke:
         packages:
         - default-mpi
-        - kripke-omp
+        - kripke
+        - '{modifier_package_name}'

repo/blt/package.py

@@ -0,0 +1,11 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from spack.package import *
+from spack.pkg.builtin.blt import Blt as BuiltinBlt
+
+
+class Blt(BuiltinBlt):
+    version("0.6.2", sha256="84b663162957c1fe0e896ac8e94cbf2b6def4a152ccfa12a293db14fb25191c8")

repo/camp/libstdc++-13-missing-header.patch

@@ -0,0 +1,24 @@
+diff --git a/include/camp/resource.hpp b/include/camp/resource.hpp
+index 58df206..763aa20 100644
+--- a/include/camp/resource.hpp
++++ b/include/camp/resource.hpp
+@@ -12,6 +12,7 @@ http://github.com/llnl/camp
+ #define __CAMP_RESOURCE_HPP
+
+ #include <cstring>
++#include <stdexcept>
+ #include <memory>
+ #include <mutex>
+ #include <type_traits>
+diff --git a/include/camp/resource/host.hpp b/include/camp/resource/host.hpp
+index c765175..4428c57 100644
+--- a/include/camp/resource/host.hpp
++++ b/include/camp/resource/host.hpp
+@@ -13,6 +13,7 @@ http://github.com/llnl/camp
+
+ #include "camp/resource/event.hpp"
+ #include "camp/resource/platform.hpp"
++#include <cstdlib>
+
+ namespace camp
+ {

repo/camp/package.py

@@ -0,0 +1,108 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import glob
+
+from spack.package import *
+
+
+def hip_repair_options(options, spec):
+    # there is only one dir like this, but the version component is unknown
+    options.append(
+        "-DHIP_CLANG_INCLUDE_PATH="
+        + glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0]
+    )
+
+
+def hip_repair_cache(options, spec):
+    # there is only one dir like this, but the version component is unknown
+    options.append(
+        cmake_cache_path(
+            "HIP_CLANG_INCLUDE_PATH",
+            glob.glob("{}/lib/clang/*/include".format(spec["llvm-amdgpu"].prefix))[0],
+        )
+    )
+
+
+class Camp(CMakePackage, CudaPackage, ROCmPackage):
+    """
+    Compiler agnostic metaprogramming library providing concepts,
+    type operations and tuples for C++ and cuda
+    """
+
+    homepage = "https://github.com/LLNL/camp"
+    git = "https://github.com/LLNL/camp.git"
+    url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz"
+
+    maintainers("trws")
+
+    license("BSD-3-Clause")
+
+    version("main", branch="main", submodules="True")
+    version(
+        "2024.02.1",
+        tag="v2024.02.1",
+        commit="79c320fa09db987923b56884afdc9f82f4b70fc4",
+        submodules=False,
+    )
+    version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02")
+    version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb")
+    version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721")
+    version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb")
+    version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7")
+    version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819")
+    version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc")
+
+    # TODO: figure out gtest dependency and then set this default True.
+    variant("tests", default=False, description="Build tests")
+    variant("openmp", default=False, description="Build OpenMP support")
+
+    depends_on("cub", when="+cuda")
+
+    depends_on("blt", type="build")
+    depends_on("[email protected]:0.5.3", type="build", when="@2022.03.0:2024.01")
+    depends_on("[email protected]", type="build", when="@2024.02:")
+
+    patch("libstdc++-13-missing-header.patch", when="@:2022.10")
+
+    conflicts("^blt@:0.3.6", when="+rocm")
+
+    def cmake_args(self):
+        spec = self.spec
+
+        options = []
+
+        options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix))
+
+        options.append("-DENABLE_OPENMP=" + ("On" if "+openmp" in spec else "Off"))
+        if "+cuda" in spec:
+            options.extend(
+                ["-DENABLE_CUDA=ON", "-DCUDA_TOOLKIT_ROOT_DIR=%s" % (spec["cuda"].prefix)]
+            )
+
+            if not spec.satisfies("cuda_arch=none"):
+                cuda_arch = spec.variants["cuda_arch"].value
+                options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0]))
+                options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0]))
+                flag = "-arch sm_{0}".format(cuda_arch[0])
+                options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag))
+        else:
+            options.append("-DENABLE_CUDA=OFF")
+
+        if "+rocm" in spec:
+            options.extend(["-DENABLE_HIP=ON", "-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)])
+
+            hip_repair_options(options, spec)
+
+            archs = self.spec.variants["amdgpu_target"].value
+            if archs != "none":
+                arch_str = ",".join(archs)
+                options.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(arch_str))
+        else:
+            options.append("-DENABLE_HIP=OFF")
+
+        options.append(self.define_from_variant("ENABLE_TESTS", "tests"))
+
+        return options