Skip to content

Commit

Permalink
Merge branch 'develop' into pearce8-elf
Browse files Browse the repository at this point in the history
  • Loading branch information
pearce8 authored Jan 8, 2025
2 parents c4e6a42 + 8716106 commit 0d0c5ae
Show file tree
Hide file tree
Showing 20 changed files with 340 additions and 127 deletions.
48 changes: 38 additions & 10 deletions .github/workflows/run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
- name: Upload Workspace Archive as CI Artifact
if: always()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
with:
name: workspace-archive
path: './workspace/saxpy/openmp/nosite-x86_64/workspace/archive/**'
Expand Down Expand Up @@ -478,26 +478,54 @@ jobs:
--disable-logger \
workspace setup --dry-run
- name: Dry run dynamic remhos/mpi with dynamic llnl-cluster ruby
- name: Dry run dynamic remhos/mpi with dynamic Ruby
run: |
./bin/benchpark experiment init --dest=remhos-ruby remhos
./bin/benchpark setup ./remhos-ruby ./ruby-system workspace/
system_id=$(./bin/benchpark system id ./ruby-system)
./bin/benchpark system init --dest=cluster-system1 llnl-cluster compiler=gcc lapack=intel-oneapi-mkl blas=intel-oneapi-mkl
system_id=$(./bin/benchpark system id ./cluster-system1)
./bin/benchpark experiment init --dest=remhos-mpi-ruby remhos ~cuda~rocm caliper=mpi,time
./bin/benchpark setup ./remhos-mpi-ruby ./cluster-system1 workspace/
. workspace/setup.sh
ramble \
--workspace-dir workspace/remhos-ruby/$system_id/workspace \
--workspace-dir "workspace/remhos-mpi-ruby/$system_id/workspace" \
--disable-progress-bar \
--disable-logger \
workspace setup --dry-run
- name: Dry run dynamic remhos/mpi with dynamic Tioga
run: |
./bin/benchpark experiment init --dest=remhos-tioga remhos
./bin/benchpark setup remhos-tioga ./tioga-system workspace/
system_id=$(./bin/benchpark system id ./tioga-system)
./bin/benchpark system init --dest=tioga-system4 llnl-elcapitan rocm=5.5.1 +gtl compiler=cce lapack=intel-oneapi-mkl blas=intel-oneapi-mkl
system_id=$(./bin/benchpark system id ./tioga-system4)
./bin/benchpark experiment init --dest=remhos-mpi-tioga remhos ~cuda~rocm caliper=mpi,time
./bin/benchpark setup ./remhos-mpi-tioga ./tioga-system4 workspace/
. workspace/setup.sh
ramble \
--workspace-dir "workspace/remhos-mpi-tioga/$system_id/workspace" \
--disable-progress-bar \
--disable-logger \
workspace setup --dry-run
- name: Dry run dynamic remhos/cuda with dynamic Sierra
run: |
./bin/benchpark system init --dest=sierra-system2 llnl-sierra cuda=11-8-0 compiler=clang-ibm lapack=essl blas=cublas
system_id=$(./bin/benchpark system id ./sierra-system2)
./bin/benchpark experiment init --dest=remhos-cuda remhos +cuda~rocm caliper=mpi,time
./bin/benchpark setup ./remhos-cuda ./sierra-system2 workspace/
. workspace/setup.sh
ramble \
--workspace-dir "workspace/remhos-cuda/$system_id/workspace" \
--disable-progress-bar \
--disable-logger \
workspace setup --dry-run
- name: Dry run dynamic remhos/rocm with dynamic Tioga
run: |
./bin/benchpark system init --dest=tioga-system5 llnl-elcapitan rocm=5.5.1 +gtl compiler=cce lapack=cray-libsci blas=rocblas
system_id=$(./bin/benchpark system id ./tioga-system5)
./bin/benchpark experiment init --dest=remhos-rocm remhos ~cuda+rocm caliper=mpi,time
./bin/benchpark setup ./remhos-rocm ./tioga-system5 workspace/
. workspace/setup.sh
ramble \
--workspace-dir workspace/remhos-tioga/$system_id/workspace \
--workspace-dir "workspace/remhos-rocm/$system_id/workspace" \
--disable-progress-bar \
--disable-logger \
workspace setup --dry-run
Expand Down
19 changes: 9 additions & 10 deletions experiments/amg2023/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,22 +140,21 @@ def compute_spack_section(self):
system_specs = {}
system_specs["compiler"] = "default-compiler"
system_specs["mpi"] = "default-mpi"
system_specs["lapack"] = "lapack"
system_specs["lapack"] = "default-lapack"
system_specs["blas"] = "default-blas"

# set package spack specs
# empty package_specs value implies external package
self.add_spack_spec(system_specs["mpi"])

if self.spec.satisfies("+cuda"):
system_specs["cuda_version"] = "{default_cuda_version}"
system_specs["cuda_arch"] = "{cuda_arch}"
system_specs["blas"] = "cublas-cuda"
if self.spec.satisfies("+rocm"):
elif self.spec.satisfies("+rocm"):
system_specs["rocm_arch"] = "{rocm_arch}"
system_specs["blas"] = "blas-rocm"

# set package spack specs
if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
# empty package_specs value implies external package
self.add_spack_spec(system_specs["blas"])
# empty package_specs value implies external package
self.add_spack_spec(system_specs["mpi"])
# empty package_specs value implies external package
self.add_spack_spec(system_specs["blas"])
self.add_spack_spec(system_specs["lapack"])

self.add_spack_spec(
Expand Down
59 changes: 43 additions & 16 deletions experiments/remhos/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,16 @@
from benchpark.experiment import Experiment
from benchpark.scaling import StrongScaling
from benchpark.expr.builtin.caliper import Caliper
from benchpark.cuda import CudaExperiment
from benchpark.rocm import ROCmExperiment


class Remhos(
Experiment,
StrongScaling,
Caliper,
CudaExperiment,
ROCmExperiment,
):

variant(
Expand All @@ -25,8 +29,8 @@ class Remhos(

variant(
"version",
default="gpu-fom",
values=("1.0", "develop", "gpu-fom"),
default="gpu-opt",
values=("1.0", "develop", "gpu-fom", "gpu-opt"),
description="app version",
)

Expand All @@ -44,24 +48,40 @@ def compute_applications_section(self):
f"Only one type of scaling per experiment is allowed for application package {self.name}"
)

# Number of initial nodes
num_nodes = {"n_nodes": 1}
n_resources = {"n_nodes": 8}
# problem_size = {"epm": 512}
device = "n_ranks"

if self.spec.satisfies("+cuda"):
self.add_experiment_variable("device", "cuda", True)
elif self.spec.satisfies("+rocm"):
self.add_experiment_variable("device", "hip", True)
if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
device = "n_gpus"
else:
self.add_experiment_variable(
"n_ranks", "{sys_cores_per_node} * {n_nodes}", True
)
self.add_experiment_variable("device", "cpu", True)

if self.spec.satisfies("+single_node"):
for pk, pv in num_nodes.items():
self.add_experiment_variable(pk, pv, True)
for pk, pv in n_resources.items():
self.add_experiment_variable(device, pv, True)

elif self.spec.satisfies("+strong"):
scaled_variables = self.generate_strong_scaling_params(
{tuple(num_nodes.keys()): list(num_nodes.values())},
{tuple(n_resources.keys()): list(n_resources.values())},
int(self.spec.variants["scaling-factor"][0]),
int(self.spec.variants["scaling-iterations"][0]),
)
for pk, pv in scaled_variables.items():
self.add_experiment_variable(pk, pv, True)

self.add_experiment_variable(
"n_ranks", "{sys_cores_per_node} * {n_nodes}", True
)
num_resources = scaled_variables["n_nodes"]
self.add_experiment_variable(device, num_resources, True)
if self.spec.satisfies("+cuda"):
self.add_experiment_variable("arch", "CUDA")
elif self.spec.satisfies("+rocm"):
self.add_experiment_variable("arch", "HIP")

def compute_spack_section(self):
# get package version
Expand All @@ -72,16 +92,23 @@ def compute_spack_section(self):
system_specs = {}
system_specs["compiler"] = "default-compiler"
system_specs["mpi"] = "default-mpi"
system_specs["lapack"] = "default-lapack"
system_specs["blas"] = "default-blas"

# set package spack specs
# empty package_specs value implies external package
self.add_spack_spec(system_specs["mpi"])
# self.add_spack_spec(system_specs["blas"])

if self.spec.satisfies("+cuda"):
system_specs["cuda_version"] = "{default_cuda_version}"
system_specs["cuda_arch"] = "{cuda_arch}"
elif self.spec.satisfies("+rocm"):
system_specs["rocm_arch"] = "{rocm_arch}"

# empty package_specs value implies external package
self.add_spack_spec(system_specs["blas"])
self.add_spack_spec(system_specs["lapack"])

self.add_spack_spec(
self.name, [f"remhos@{app_version} +metis", system_specs["compiler"]]
)
self.add_spack_spec(
"hypre",
["[email protected] +mpi+openmp+mixedint~fortran", system_specs["compiler"]],
)
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ software:
pkg_spec: [email protected]
default-mpi:
pkg_spec: [email protected]
default-blas:
pkg_spec: [email protected]
default-lapack:
pkg_spec: [email protected]
compiler-gcc:
pkg_spec: [email protected]
compiler-intel:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ software:
pkg_spec: gcc@=11.2.1
default-mpi:
pkg_spec: [email protected]
default-blas:
pkg_spec: cublas@{default_cuda_version}
default-lapack:
pkg_spec: cublas@{default_cuda_version}
compiler-gcc:
pkg_spec: gcc@=11.2.1
compiler-clang:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ software:
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
default-mpi:
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
default-blas:
pkg_spec: cublas@{default_cuda_version}
default-lapack:
pkg_spec: cublas@{default_cuda_version}
compiler-xl:
pkg_spec: [email protected]{default_cuda_version}-gcc-11.2.1
mpi-xl:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,9 @@ packages:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/hiprand
prefix: /opt/rocm-5.4.3
- spec: [email protected]
prefix: /opt/rocm-5.5.1/hiprand
prefix: /opt/rocm-5.5.1
rocsparse:
buildable: false
externals:
Expand Down Expand Up @@ -186,6 +186,13 @@ packages:
prefix: /opt/rocm-5.4.3/
- spec: [email protected]
prefix: /opt/rocm-5.5.1/
hiprand:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
- spec: [email protected]
prefix: /opt/rocm-5.5.1
hipsparse:
buildable: false
externals:
Expand Down
18 changes: 18 additions & 0 deletions repo/essl/package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Copyright 2023 Lawrence Livermore National Security, LLC and other
# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: Apache-2.0

from spack.package import *
from spack.pkg.builtin.essl import Essl as BuiltinEssl


class Essl(BuiltinEssl):
@property
def lapack_libs(self):
essl_libs = super(Essl, self).lapack_libs

essl_libs += find_libraries(
["libessl", "libessl6464", "libesslsmp", "libesslsmp6464"], root=self.prefix.lib64, shared=True
)
return essl_libs
61 changes: 61 additions & 0 deletions repo/mfem/package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Copyright 2023 Lawrence Livermore National Security, LLC and other
# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: Apache-2.0

import os

from spack.package import *
from spack.pkg.builtin.mfem import Mfem as BuiltinMfem


class Mfem(BuiltinMfem):
# variant("rocm", default=False, description="Enable ROCm support")
# depends_on("rocblas", when="+rocm")
# depends_on("rocsolver", when="+rocm")

requires("+rocm", when="^rocblas")
requires("+rocm", when="^rocsolver")
requires("+caliper", when="^hypre+caliper")
requires("+mpi", when="^hypre+mpi")

depends_on("hiprand", when="+rocm")
depends_on("hipsparse", when="+rocm")

version("4.4_comm_cali", branch="comm_cali", submodules=False, git="https://github.com/gracenansamba/mfem.git")

variant("caliper", default=False, description="Build Caliper support")

depends_on("caliper", when="+caliper")
depends_on("adiak", when="+caliper")

compiler_to_cpe_name = {
"cce": "cray",
"gcc": "gnu",
}

#def get_make_config_options(self, spec, prefix):
# def yes_no(varstr):
# return "YES" if varstr in self.spec else "NO"
# options = super(Mfem, self).get_make_config_options(spec, prefix)
# caliper_opt = ["MFEM_USE_CALIPER=%s" % yes_no("+caliper"), ]
# return options + caliper_opt




#cal_dir=spec["caliper"].prefix
#targets.append("CALIPER_DIR=%s" % spec["caliper"].prefix)
#targets.append("ADIAK_DIR=%s" % spec["adiak"].prefix)
def get_make_config_options(self, spec, prefix):
def yes_no(varstr):
return "YES" if varstr in self.spec else "NO"
options = super(Mfem, self).get_make_config_options(spec, prefix)
options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper"))
if "+caliper" in self.spec:
options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix)
options.append("MFEM_USE_ADIAK=%s" % yes_no("+adiak"))
options.append("ADIAK_DIR=%s" % self.spec["adiak"].prefix)

return options

10 changes: 7 additions & 3 deletions repo/remhos/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ class Remhos(ExecutableApplication):
"""Remhos benchmark"""
name = "remhos"
#TODO: add -ms flag once it's implemented
executable('2d', 'remhos'+' -m {remhos}/data/inline-quad.mesh'+' -p 14'+' -rs {rs2d}'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -no-vis', use_mpi=True)
executable('3d', 'remhos'+' -m {remhos}/data/cube01_hex.mesh'+' -p 10'+' -rs {rs3d}'+' -o {o}'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -no-vis', use_mpi=True)
executable('2d', 'remhos'+' -dim 2 -epm 1024'+' -p 14'+' -rs {rs2d}'+' -o 3 -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
executable('3d', 'remhos'+' -dim 3 -epm 512'+' -p 10'+' -rs {rs3d}'+' -o 2'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
workload('2d', executables=['2d'])
workload('3d', executables=['3d'])

Expand Down Expand Up @@ -68,7 +68,11 @@ class Remhos(ExecutableApplication):
workload_variable('ms', default='5',
description='ms',
workloads=['2d','3d'])
#FOM_regex=r'(?<=Merit)\s+[\+\-]*[0-9]*\.*[0-9]+e*[\+\-]*[0-9]*'
workload_variable('device', default='cpu',
description='cpu or cuda',
workloads=['2d','3d'])

figure_of_merit("FOM", log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'FOM:\s+(?P<fom>[0-9]*\.[0-9]*)', group_name='fom', units='megadofs x time steps / second')
#FOM_regex=r'(?<=Merit)\s+[\+\-]*[0-9]*\.*[0-9]+e*[\+\-]*[0-9]*'
success_criteria('valid', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out')

Loading

0 comments on commit 0d0c5ae

Please sign in to comment.