Remhos gpu experiment (#508)

* progress on remhos gpu

* updating progress

* updating cpu mode

* removing extra hypre variable

* most recent updates

* adding hypre package

* remhos cuda/hip support

* Fix for blas/lapack on lassen

* HIP backend

* lint

* lint

* lint

* lint

* Add ibm essl option for blas on sierra

* Fix workflows

* Add ibm essl option for blas on sierra

* updating run parameters

* remove hip patch

* dry runs

* Remove amg2023 changes

* setting github url back to upstream

---------

Co-authored-by: august-knox <[email protected]>
Co-authored-by: Riyaz Haque <[email protected]>
Co-authored-by: pearce8 <[email protected]>

.github/workflows/run.yml

@@ -478,26 +478,54 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
-      - name: Dry run dynamic remhos/mpi with dynamic llnl-cluster ruby
+      - name: Dry run dynamic remhos/mpi with dynamic Ruby
         run: |
-          ./bin/benchpark experiment init --dest=remhos-ruby remhos
-          ./bin/benchpark setup ./remhos-ruby ./ruby-system workspace/
-          system_id=$(./bin/benchpark system id ./ruby-system)
+          ./bin/benchpark system init --dest=cluster-system1 llnl-cluster compiler=gcc lapack=intel-oneapi-mkl blas=intel-oneapi-mkl
+          system_id=$(./bin/benchpark system id ./cluster-system1)
+          ./bin/benchpark experiment init --dest=remhos-mpi-ruby remhos ~cuda~rocm caliper=mpi,time
+          ./bin/benchpark setup ./remhos-mpi-ruby ./cluster-system1 workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/remhos-ruby/$system_id/workspace \
+            --workspace-dir "workspace/remhos-mpi-ruby/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run dynamic remhos/mpi with dynamic Tioga
         run: |
-          ./bin/benchpark experiment init --dest=remhos-tioga remhos
-          ./bin/benchpark setup remhos-tioga ./tioga-system workspace/
-          system_id=$(./bin/benchpark system id ./tioga-system)
+          ./bin/benchpark system init --dest=tioga-system4 llnl-elcapitan rocm=5.5.1 +gtl compiler=cce lapack=intel-oneapi-mkl blas=intel-oneapi-mkl
+          system_id=$(./bin/benchpark system id ./tioga-system4)
+          ./bin/benchpark experiment init --dest=remhos-mpi-tioga remhos ~cuda~rocm caliper=mpi,time
+          ./bin/benchpark setup ./remhos-mpi-tioga ./tioga-system4 workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/remhos-mpi-tioga/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic remhos/cuda with dynamic Sierra
+        run: |
+          ./bin/benchpark system init --dest=sierra-system2 llnl-sierra cuda=11-8-0 compiler=clang-ibm lapack=essl blas=cublas
+          system_id=$(./bin/benchpark system id ./sierra-system2)
+          ./bin/benchpark experiment init --dest=remhos-cuda remhos +cuda~rocm caliper=mpi,time
+          ./bin/benchpark setup ./remhos-cuda ./sierra-system2 workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/remhos-cuda/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic remhos/rocm with dynamic Tioga
+        run: |
+          ./bin/benchpark system init --dest=tioga-system5 llnl-elcapitan rocm=5.5.1 +gtl compiler=cce lapack=cray-libsci blas=rocblas
+          system_id=$(./bin/benchpark system id ./tioga-system5)
+          ./bin/benchpark experiment init --dest=remhos-rocm remhos ~cuda+rocm caliper=mpi,time
+          ./bin/benchpark setup ./remhos-rocm ./tioga-system5 workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/remhos-tioga/$system_id/workspace \
+            --workspace-dir "workspace/remhos-rocm/$system_id/workspace" \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

experiments/remhos/experiment.py

@@ -8,12 +8,16 @@
 from benchpark.experiment import Experiment
 from benchpark.scaling import StrongScaling
 from benchpark.expr.builtin.caliper import Caliper
+from benchpark.cuda import CudaExperiment
+from benchpark.rocm import ROCmExperiment
 
 
 class Remhos(
     Experiment,
     StrongScaling,
     Caliper,
+    CudaExperiment,
+    ROCmExperiment,
 ):
 
     variant(
@@ -25,8 +29,8 @@ class Remhos(
 
     variant(
         "version",
-        default="gpu-fom",
-        values=("1.0", "develop", "gpu-fom"),
+        default="gpu-opt",
+        values=("1.0", "develop", "gpu-fom", "gpu-opt"),
         description="app version",
     )
 
@@ -44,24 +48,40 @@ def compute_applications_section(self):
                 f"Only one type of scaling per experiment is allowed for application package {self.name}"
             )
 
-        # Number of initial nodes
-        num_nodes = {"n_nodes": 1}
+        n_resources = {"n_nodes": 8}
+        # problem_size = {"epm": 512}
+        device = "n_ranks"
+
+        if self.spec.satisfies("+cuda"):
+            self.add_experiment_variable("device", "cuda", True)
+        elif self.spec.satisfies("+rocm"):
+            self.add_experiment_variable("device", "hip", True)
+        if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
+            device = "n_gpus"
+        else:
+            self.add_experiment_variable(
+                "n_ranks", "{sys_cores_per_node} * {n_nodes}", True
+            )
+            self.add_experiment_variable("device", "cpu", True)
 
         if self.spec.satisfies("+single_node"):
-            for pk, pv in num_nodes.items():
-                self.add_experiment_variable(pk, pv, True)
+            for pk, pv in n_resources.items():
+                self.add_experiment_variable(device, pv, True)
+
         elif self.spec.satisfies("+strong"):
             scaled_variables = self.generate_strong_scaling_params(
-                {tuple(num_nodes.keys()): list(num_nodes.values())},
+                {tuple(n_resources.keys()): list(n_resources.values())},
                 int(self.spec.variants["scaling-factor"][0]),
                 int(self.spec.variants["scaling-iterations"][0]),
             )
             for pk, pv in scaled_variables.items():
                 self.add_experiment_variable(pk, pv, True)
-
-        self.add_experiment_variable(
-            "n_ranks", "{sys_cores_per_node} * {n_nodes}", True
-        )
+            num_resources = scaled_variables["n_nodes"]
+            self.add_experiment_variable(device, num_resources, True)
+        if self.spec.satisfies("+cuda"):
+            self.add_experiment_variable("arch", "CUDA")
+        elif self.spec.satisfies("+rocm"):
+            self.add_experiment_variable("arch", "HIP")
 
     def compute_spack_section(self):
         # get package version
@@ -72,16 +92,23 @@ def compute_spack_section(self):
         system_specs = {}
         system_specs["compiler"] = "default-compiler"
         system_specs["mpi"] = "default-mpi"
+        system_specs["lapack"] = "default-lapack"
+        system_specs["blas"] = "default-blas"
 
         # set package spack specs
         # empty package_specs value implies external package
         self.add_spack_spec(system_specs["mpi"])
-        # self.add_spack_spec(system_specs["blas"])
+
+        if self.spec.satisfies("+cuda"):
+            system_specs["cuda_version"] = "{default_cuda_version}"
+            system_specs["cuda_arch"] = "{cuda_arch}"
+        elif self.spec.satisfies("+rocm"):
+            system_specs["rocm_arch"] = "{rocm_arch}"
+
+        # empty package_specs value implies external package
+        self.add_spack_spec(system_specs["blas"])
+        self.add_spack_spec(system_specs["lapack"])
 
         self.add_spack_spec(
             self.name, [f"remhos@{app_version} +metis", system_specs["compiler"]]
         )
-        self.add_spack_spec(
-            "hypre",
-            ["[email protected] +mpi+openmp+mixedint~fortran", system_specs["compiler"]],
-        )

legacy/systems/LLNL-Dane-DELL-sapphirerapids-OmniPath/software.yaml

@@ -9,6 +9,10 @@ software:
       pkg_spec: [email protected]
     default-mpi:
       pkg_spec: [email protected]
+    default-blas:
+      pkg_spec: [email protected]
+    default-lapack:
+      pkg_spec: [email protected]
     compiler-gcc:
       pkg_spec: [email protected]
     compiler-intel:

legacy/systems/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/software.yaml

@@ -9,6 +9,10 @@ software:
       pkg_spec: gcc@=11.2.1
     default-mpi:
       pkg_spec: [email protected]
+    default-blas:
+      pkg_spec: cublas@{default_cuda_version}
+    default-lapack:
+      pkg_spec: cublas@{default_cuda_version}
     compiler-gcc:
       pkg_spec: gcc@=11.2.1
     compiler-clang:

legacy/systems/LLNL-Sierra-IBM-power9-V100-Infiniband/software.yaml

@@ -9,6 +9,10 @@ software:
       pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
     default-mpi:
       pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
+    default-blas:
+      pkg_spec: cublas@{default_cuda_version}
+    default-lapack:
+      pkg_spec: cublas@{default_cuda_version}
     compiler-xl:
       pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
     mpi-xl:

legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml

@@ -148,9 +148,9 @@ packages:
     buildable: false
     externals:
     - spec: [email protected]
-      prefix: /opt/rocm-5.4.3/hiprand
+      prefix: /opt/rocm-5.4.3
     - spec: [email protected]
-      prefix: /opt/rocm-5.5.1/hiprand
+      prefix: /opt/rocm-5.5.1
   rocsparse:
     buildable: false
     externals:
@@ -186,6 +186,13 @@ packages:
       prefix: /opt/rocm-5.4.3/
     - spec: [email protected]
       prefix: /opt/rocm-5.5.1/
+  hiprand:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
   hipsparse:
     buildable: false
     externals:

repo/essl/package.py

@@ -0,0 +1,18 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from spack.package import *
+from spack.pkg.builtin.essl import Essl as BuiltinEssl
+
+
+class Essl(BuiltinEssl):
+    @property
+    def lapack_libs(self):
+        essl_libs = super(Essl, self).lapack_libs
+
+        essl_libs += find_libraries(
+            ["libessl", "libessl6464", "libesslsmp", "libesslsmp6464"], root=self.prefix.lib64, shared=True
+        )
+        return essl_libs

repo/mfem/package.py

@@ -0,0 +1,61 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from spack.package import *
+from spack.pkg.builtin.mfem import Mfem as BuiltinMfem
+
+
+class Mfem(BuiltinMfem):
+    # variant("rocm", default=False, description="Enable ROCm support")
+    # depends_on("rocblas", when="+rocm")
+    # depends_on("rocsolver", when="+rocm")
+
+    requires("+rocm", when="^rocblas")
+    requires("+rocm", when="^rocsolver")
+    requires("+caliper", when="^hypre+caliper")
+    requires("+mpi", when="^hypre+mpi")
+
+    depends_on("hiprand", when="+rocm")
+    depends_on("hipsparse", when="+rocm")
+
+    version("4.4_comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/mfem.git")
+
+    variant("caliper", default=False, description="Build Caliper support")
+
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak", when="+caliper")
+
+    compiler_to_cpe_name = {
+        "cce": "cray",
+        "gcc": "gnu",
+    }
+
+    #def get_make_config_options(self, spec, prefix):
+    #    def yes_no(varstr):
+    #        return "YES" if varstr in self.spec else "NO"
+    #    options = super(Mfem, self).get_make_config_options(spec, prefix)
+    #    caliper_opt = ["MFEM_USE_CALIPER=%s" % yes_no("+caliper"), ]
+    #    return options + caliper_opt
+
+
+
+
+    #cal_dir=spec["caliper"].prefix
+    #targets.append("CALIPER_DIR=%s" % spec["caliper"].prefix)
+    #targets.append("ADIAK_DIR=%s" % spec["adiak"].prefix)
+    def get_make_config_options(self, spec, prefix):
+        def yes_no(varstr):
+            return "YES" if varstr in self.spec else "NO"
+        options = super(Mfem, self).get_make_config_options(spec, prefix)
+        options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper"))
+        if "+caliper" in self.spec: 
+            options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix)
+            options.append("MFEM_USE_ADIAK=%s" % yes_no("+adiak"))
+            options.append("ADIAK_DIR=%s" % self.spec["adiak"].prefix)
+
+        return options
+

repo/remhos/application.py

@@ -12,8 +12,8 @@ class Remhos(ExecutableApplication):
     """Remhos benchmark"""
     name = "remhos"
 #TODO: add -ms flag once it's implemented
-    executable('2d', 'remhos'+' -m {remhos}/data/inline-quad.mesh'+' -p 14'+' -rs {rs2d}'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+'  -no-vis', use_mpi=True)
-    executable('3d', 'remhos'+' -m {remhos}/data/cube01_hex.mesh'+' -p 10'+' -rs {rs3d}'+' -o {o}'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -no-vis', use_mpi=True)
+    executable('2d', 'remhos'+' -dim 2 -epm 1024'+' -p 14'+' -rs {rs2d}'+' -o 3 -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
+    executable('3d', 'remhos'+' -dim 3 -epm 512'+' -p 10'+' -rs {rs3d}'+' -o 2'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
     workload('2d', executables=['2d'])
     workload('3d', executables=['3d'])
 
@@ -68,7 +68,11 @@ class Remhos(ExecutableApplication):
     workload_variable('ms', default='5',
         description='ms',
         workloads=['2d','3d'])
-    #FOM_regex=r'(?<=Merit)\s+[\+\-]*[0-9]*\.*[0-9]+e*[\+\-]*[0-9]*'
+    workload_variable('device', default='cpu',
+        description='cpu or cuda',
+        workloads=['2d','3d'])
+
     figure_of_merit("FOM", log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'FOM:\s+(?P<fom>[0-9]*\.[0-9]*)', group_name='fom', units='megadofs x time steps / second')
+    #FOM_regex=r'(?<=Merit)\s+[\+\-]*[0-9]*\.*[0-9]+e*[\+\-]*[0-9]*'
     success_criteria('valid', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out')