-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'develop' into pearce8-elf
- Loading branch information
Showing
20 changed files
with
340 additions
and
127 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,12 +8,16 @@ | |
from benchpark.experiment import Experiment | ||
from benchpark.scaling import StrongScaling | ||
from benchpark.expr.builtin.caliper import Caliper | ||
from benchpark.cuda import CudaExperiment | ||
from benchpark.rocm import ROCmExperiment | ||
|
||
|
||
class Remhos( | ||
Experiment, | ||
StrongScaling, | ||
Caliper, | ||
CudaExperiment, | ||
ROCmExperiment, | ||
): | ||
|
||
variant( | ||
|
@@ -25,8 +29,8 @@ class Remhos( | |
|
||
variant( | ||
"version", | ||
default="gpu-fom", | ||
values=("1.0", "develop", "gpu-fom"), | ||
default="gpu-opt", | ||
values=("1.0", "develop", "gpu-fom", "gpu-opt"), | ||
description="app version", | ||
) | ||
|
||
|
@@ -44,24 +48,40 @@ def compute_applications_section(self): | |
f"Only one type of scaling per experiment is allowed for application package {self.name}" | ||
) | ||
|
||
# Number of initial nodes | ||
num_nodes = {"n_nodes": 1} | ||
n_resources = {"n_nodes": 8} | ||
# problem_size = {"epm": 512} | ||
device = "n_ranks" | ||
|
||
if self.spec.satisfies("+cuda"): | ||
self.add_experiment_variable("device", "cuda", True) | ||
elif self.spec.satisfies("+rocm"): | ||
self.add_experiment_variable("device", "hip", True) | ||
if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): | ||
device = "n_gpus" | ||
else: | ||
self.add_experiment_variable( | ||
"n_ranks", "{sys_cores_per_node} * {n_nodes}", True | ||
) | ||
self.add_experiment_variable("device", "cpu", True) | ||
|
||
if self.spec.satisfies("+single_node"): | ||
for pk, pv in num_nodes.items(): | ||
self.add_experiment_variable(pk, pv, True) | ||
for pk, pv in n_resources.items(): | ||
self.add_experiment_variable(device, pv, True) | ||
|
||
elif self.spec.satisfies("+strong"): | ||
scaled_variables = self.generate_strong_scaling_params( | ||
{tuple(num_nodes.keys()): list(num_nodes.values())}, | ||
{tuple(n_resources.keys()): list(n_resources.values())}, | ||
int(self.spec.variants["scaling-factor"][0]), | ||
int(self.spec.variants["scaling-iterations"][0]), | ||
) | ||
for pk, pv in scaled_variables.items(): | ||
self.add_experiment_variable(pk, pv, True) | ||
|
||
self.add_experiment_variable( | ||
"n_ranks", "{sys_cores_per_node} * {n_nodes}", True | ||
) | ||
num_resources = scaled_variables["n_nodes"] | ||
self.add_experiment_variable(device, num_resources, True) | ||
if self.spec.satisfies("+cuda"): | ||
self.add_experiment_variable("arch", "CUDA") | ||
elif self.spec.satisfies("+rocm"): | ||
self.add_experiment_variable("arch", "HIP") | ||
|
||
def compute_spack_section(self): | ||
# get package version | ||
|
@@ -72,16 +92,23 @@ def compute_spack_section(self): | |
system_specs = {} | ||
system_specs["compiler"] = "default-compiler" | ||
system_specs["mpi"] = "default-mpi" | ||
system_specs["lapack"] = "default-lapack" | ||
system_specs["blas"] = "default-blas" | ||
|
||
# set package spack specs | ||
# empty package_specs value implies external package | ||
self.add_spack_spec(system_specs["mpi"]) | ||
# self.add_spack_spec(system_specs["blas"]) | ||
|
||
if self.spec.satisfies("+cuda"): | ||
system_specs["cuda_version"] = "{default_cuda_version}" | ||
system_specs["cuda_arch"] = "{cuda_arch}" | ||
elif self.spec.satisfies("+rocm"): | ||
system_specs["rocm_arch"] = "{rocm_arch}" | ||
|
||
# empty package_specs value implies external package | ||
self.add_spack_spec(system_specs["blas"]) | ||
self.add_spack_spec(system_specs["lapack"]) | ||
|
||
self.add_spack_spec( | ||
self.name, [f"remhos@{app_version} +metis", system_specs["compiler"]] | ||
) | ||
self.add_spack_spec( | ||
"hypre", | ||
["[email protected] +mpi+openmp+mixedint~fortran", system_specs["compiler"]], | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,6 +9,10 @@ software: | |
pkg_spec: [email protected] | ||
default-mpi: | ||
pkg_spec: [email protected] | ||
default-blas: | ||
pkg_spec: [email protected] | ||
default-lapack: | ||
pkg_spec: [email protected] | ||
compiler-gcc: | ||
pkg_spec: [email protected] | ||
compiler-intel: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,6 +9,10 @@ software: | |
pkg_spec: gcc@=11.2.1 | ||
default-mpi: | ||
pkg_spec: [email protected] | ||
default-blas: | ||
pkg_spec: cublas@{default_cuda_version} | ||
default-lapack: | ||
pkg_spec: cublas@{default_cuda_version} | ||
compiler-gcc: | ||
pkg_spec: gcc@=11.2.1 | ||
compiler-clang: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,6 +9,10 @@ software: | |
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1 | ||
default-mpi: | ||
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1 | ||
default-blas: | ||
pkg_spec: cublas@{default_cuda_version} | ||
default-lapack: | ||
pkg_spec: cublas@{default_cuda_version} | ||
compiler-xl: | ||
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1 | ||
mpi-xl: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -148,9 +148,9 @@ packages: | |
buildable: false | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/rocm-5.4.3/hiprand | ||
prefix: /opt/rocm-5.4.3 | ||
- spec: [email protected] | ||
prefix: /opt/rocm-5.5.1/hiprand | ||
prefix: /opt/rocm-5.5.1 | ||
rocsparse: | ||
buildable: false | ||
externals: | ||
|
@@ -186,6 +186,13 @@ packages: | |
prefix: /opt/rocm-5.4.3/ | ||
- spec: [email protected] | ||
prefix: /opt/rocm-5.5.1/ | ||
hiprand: | ||
buildable: false | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/rocm-5.4.3 | ||
- spec: [email protected] | ||
prefix: /opt/rocm-5.5.1 | ||
hipsparse: | ||
buildable: false | ||
externals: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
# Copyright 2023 Lawrence Livermore National Security, LLC and other | ||
# Benchpark Project Developers. See the top-level COPYRIGHT file for details. | ||
# | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
from spack.package import * | ||
from spack.pkg.builtin.essl import Essl as BuiltinEssl | ||
|
||
|
||
class Essl(BuiltinEssl): | ||
@property | ||
def lapack_libs(self): | ||
essl_libs = super(Essl, self).lapack_libs | ||
|
||
essl_libs += find_libraries( | ||
["libessl", "libessl6464", "libesslsmp", "libesslsmp6464"], root=self.prefix.lib64, shared=True | ||
) | ||
return essl_libs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
# Copyright 2023 Lawrence Livermore National Security, LLC and other | ||
# Benchpark Project Developers. See the top-level COPYRIGHT file for details. | ||
# | ||
# SPDX-License-Identifier: Apache-2.0 | ||
|
||
import os | ||
|
||
from spack.package import * | ||
from spack.pkg.builtin.mfem import Mfem as BuiltinMfem | ||
|
||
|
||
class Mfem(BuiltinMfem): | ||
# variant("rocm", default=False, description="Enable ROCm support") | ||
# depends_on("rocblas", when="+rocm") | ||
# depends_on("rocsolver", when="+rocm") | ||
|
||
requires("+rocm", when="^rocblas") | ||
requires("+rocm", when="^rocsolver") | ||
requires("+caliper", when="^hypre+caliper") | ||
requires("+mpi", when="^hypre+mpi") | ||
|
||
depends_on("hiprand", when="+rocm") | ||
depends_on("hipsparse", when="+rocm") | ||
|
||
version("4.4_comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/mfem.git") | ||
|
||
variant("caliper", default=False, description="Build Caliper support") | ||
|
||
depends_on("caliper", when="+caliper") | ||
depends_on("adiak", when="+caliper") | ||
|
||
compiler_to_cpe_name = { | ||
"cce": "cray", | ||
"gcc": "gnu", | ||
} | ||
|
||
#def get_make_config_options(self, spec, prefix): | ||
# def yes_no(varstr): | ||
# return "YES" if varstr in self.spec else "NO" | ||
# options = super(Mfem, self).get_make_config_options(spec, prefix) | ||
# caliper_opt = ["MFEM_USE_CALIPER=%s" % yes_no("+caliper"), ] | ||
# return options + caliper_opt | ||
|
||
|
||
|
||
|
||
#cal_dir=spec["caliper"].prefix | ||
#targets.append("CALIPER_DIR=%s" % spec["caliper"].prefix) | ||
#targets.append("ADIAK_DIR=%s" % spec["adiak"].prefix) | ||
def get_make_config_options(self, spec, prefix): | ||
def yes_no(varstr): | ||
return "YES" if varstr in self.spec else "NO" | ||
options = super(Mfem, self).get_make_config_options(spec, prefix) | ||
options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper")) | ||
if "+caliper" in self.spec: | ||
options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix) | ||
options.append("MFEM_USE_ADIAK=%s" % yes_no("+adiak")) | ||
options.append("ADIAK_DIR=%s" % self.spec["adiak"].prefix) | ||
|
||
return options | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.