From 498fc4d49d2dad97444dd49aadf86450f3545fb0 Mon Sep 17 00:00:00 2001 From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:54:44 -0800 Subject: [PATCH] lammps experiment.py (#446) * lammps experiment.py * lint formatiing * lint formatiing * Turn off gpu aware mpi for rocm * Enable gtl in lammps * Change lammps legacy ramble configs * Add lammps dry-runs * undo to merge in develop * reapply path changes --------- Co-authored-by: Riyaz Haque Co-authored-by: pearce8 --- .github/workflows/run.yml | 46 +++++++++- experiments/lammps/experiment.py | 90 +++++++++++++++++++ legacy/experiments/lammps/openmp/ramble.yaml | 75 +++++++--------- legacy/experiments/lammps/rocm/ramble.yaml | 79 +++++++--------- .../auxiliary_software_files/compilers.yaml | 4 +- .../software.yaml | 2 +- repo/lammps/package.py | 12 +++ .../compilers/rocm/00-rocm-551-compilers.yaml | 4 +- 8 files changed, 212 insertions(+), 100 deletions(-) create mode 100644 experiments/lammps/experiment.py diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 6885c420b..69fbd2384 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -243,7 +243,17 @@ jobs: --disable-logger \ workspace setup --dry-run - - name: Dry run lammps/rocm on Tioga with allocation modifier + - name: Dry run lammps/openmp with static Ruby + run: | + ./bin/benchpark setup lammps/openmp LLNL-Ruby-icelake-OmniPath workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir workspace/lammps/openmp/LLNL-Ruby-icelake-OmniPath/workspace \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + + - name: Dry run lammps/rocm with static Tioga run: | ./bin/benchpark setup lammps/rocm LLNL-Tioga-HPECray-zen3-MI250X-Slingshot workspace/ . workspace/setup.sh @@ -253,6 +263,40 @@ jobs: --disable-logger \ workspace setup --dry-run + - name: Dry run dynamic lammps/openmp on static Ruby + run: | + ./bin/benchpark experiment init --dest=lammps-openmp lammps+openmp + ./bin/benchpark setup ./lammps-openmp LLNL-Ruby-icelake-OmniPath workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir workspace/lammps-openmp/LLNL-Ruby-icelake-OmniPath/workspace \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + + - name: Dry run dynamic lammps/rocm on static Tioga + run: | + ./bin/benchpark experiment init --dest=lammps-rocm lammps+rocm + ./bin/benchpark setup ./lammps-rocm LLNL-Tioga-HPECray-zen3-MI250X-Slingshot workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir workspace/lammps-rocm/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/workspace \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + + - name: Dry run dynamic lammps/rocm with dynamic Tioga + run: | + ./bin/benchpark experiment init --dest=lammps-rocm-tioga lammps+rocm + ./bin/benchpark setup lammps-rocm-tioga ./tioga-system workspace/ + system_id=$(./bin/benchpark system id ./tioga-system) + . workspace/setup.sh + ramble \ + --workspace-dir workspace/lammps-rocm-tioga/$system_id/workspace \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + - name: Dry run hpl/openmp with Caliper-topdown modifier on Magma run: | ./bin/benchpark setup hpl/openmp --modifier=caliper-topdown LLNL-Magma-Penguin-icelake-OmniPath workspace/ diff --git a/experiments/lammps/experiment.py b/experiments/lammps/experiment.py new file mode 100644 index 000000000..602becbcc --- /dev/null +++ b/experiments/lammps/experiment.py @@ -0,0 +1,90 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from benchpark.directives import variant +from benchpark.experiment import Experiment +from benchpark.openmp import OpenMPExperiment +from benchpark.rocm import ROCmExperiment + + +class Lammps( + Experiment, + OpenMPExperiment, + ROCmExperiment, +): + variant( + "workload", + default="hns-reaxff", + values=("hns-reaxff", "lj", "eam", "chain", "chute", "rhodo"), + description="workloads", + ) + + variant( + "version", + default="20231121", + description="app version", + ) + + def compute_applications_section(self): + if self.spec.satisfies("+openmp"): + problem_sizes = {"x": 8, "y": 8, "z": 8} + kokkos_mode = "t {n_threads_per_proc}" + kokkos_gpu_aware = "off" + kokkos_comm = "host" + elif self.spec.satisfies("+rocm"): + problem_sizes = {"x": 20, "y": 40, "z": 32} + kokkos_mode = "g 1" + kokkos_gpu_aware = "on" + kokkos_comm = "device" + + for nk, nv in problem_sizes.items(): + self.add_experiment_variable(nk, nv, True) + + input_sizes = " ".join(f"-v {k} {{{k}}}" for k in problem_sizes.keys()) + + if self.spec.satisfies("+openmp"): + self.add_experiment_variable("n_nodes", 1, True) + self.add_experiment_variable("n_ranks_per_node", 36, True) + self.add_experiment_variable("n_threads_per_proc", 1, True) + elif self.spec.satisfies("+rocm"): + self.add_experiment_variable("n_nodes", 8, True) + self.add_experiment_variable("n_ranks_per_node", 8, True) + self.add_experiment_variable("n_gpus", 64, True) + + self.add_experiment_variable("timesteps", 100, False) + self.add_experiment_variable("input_file", "{input_path}/in.reaxc.hns", False) + self.add_experiment_variable( + "lammps_flags", + f"{input_sizes} -k on {kokkos_mode} -sf kk -pk kokkos gpu/aware {kokkos_gpu_aware} neigh half comm {kokkos_comm} neigh/qeq full newton on -nocite", + False, + ) + + def compute_spack_section(self): + # get package version + app_version = self.spec.variants["version"][0] + + # get system config options + # TODO: Get compiler/mpi/package handles directly from system.py + system_specs = {} + system_specs["compiler"] = "default-compiler" + system_specs["mpi"] = "default-mpi" + if self.spec.satisfies("+rocm"): + system_specs["rocm_arch"] = "{rocm_arch}" + system_specs["blas"] = "blas-rocm" + + # set package spack specs + if self.spec.satisfies("+rocm"): + # empty package_specs value implies external package + self.add_spack_spec(system_specs["blas"]) + # empty package_specs value implies external package + self.add_spack_spec(system_specs["mpi"]) + + self.add_spack_spec( + self.name, + [ + f"lammps@{app_version} +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff lammps_sizes=bigbig ", + system_specs["compiler"], + ], + ) diff --git a/legacy/experiments/lammps/openmp/ramble.yaml b/legacy/experiments/lammps/openmp/ramble.yaml index 6ce9626c3..9269cd58e 100644 --- a/legacy/experiments/lammps/openmp/ramble.yaml +++ b/legacy/experiments/lammps/openmp/ramble.yaml @@ -1,59 +1,44 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - ramble: - include: - - ./configs/software.yaml - - ./configs/variables.yaml - - config: - deprecated: true - spack_flags: - install: '--add --keep-stage' - concretize: '-U -f' - - modifiers: - - name: allocation - applications: lammps: workloads: hns-reaxff: - variables: - size_name: ['medium'] - size_x: [2] - size_y: [2] - size_z: [2] - scaling_nodes: [1] - n_nodes: '{scaling_nodes}' - n_threads_per_proc: '1' - lammps_flags: '-v x {size_x} -v y {size_y} -v z {size_z}' experiments: - scaling_{n_nodes}nodes_{size_name}: + lammps_hns-reaxff_single_node_openmp_{x}_{y}_{z}_{n_nodes}_{n_ranks_per_node}_{n_threads_per_proc}: + exclude: {} + matrix: [] + variables: + input_file: '{input_path}/in.reaxc.hns' + lammps_flags: -v x {x} -v y {y} -v z {z} -k on t {n_threads_per_proc} + -sf kk -pk kokkos gpu/aware off neigh half comm host neigh/qeq full + newton on -nocite + n_nodes: 1 + n_ranks_per_node: 36 + n_threads_per_proc: 1 + timesteps: 100 + x: 8 + y: 8 + z: 8 variants: package_manager: spack - variables: - env_name: lammps - n_ranks_per_node: ['36'] - zips: - problems: - - size_name - - size_x - - size_y - - size_z - matrix: - - problems - - scaling_nodes + zips: {} + config: + deprecated: true + spack_flags: + concretize: -U -f + install: --add --keep-stage + include: + - ./configs + modifiers: + - name: allocation software: - packages: - lammps: - pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+openmp+openmp-package+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff - compiler: default-compiler environments: lammps: packages: - - lapack - default-mpi - lammps + packages: + lammps: + compiler: default-compiler + pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff + lammps_sizes=bigbig +openmp ~rocm diff --git a/legacy/experiments/lammps/rocm/ramble.yaml b/legacy/experiments/lammps/rocm/ramble.yaml index 713bf81eb..5d0586984 100644 --- a/legacy/experiments/lammps/rocm/ramble.yaml +++ b/legacy/experiments/lammps/rocm/ramble.yaml @@ -1,63 +1,44 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - ramble: - include: - - ./configs/software.yaml - - ./configs/variables.yaml - - ./configs/modifier.yaml - - config: - deprecated: true - spack_flags: - install: '--add --keep-stage' - concretize: '-U -f' - - modifiers: - - name: allocation - applications: lammps: workloads: hns-reaxff: - variables: - scaling_nodes: ['8'] - n_ranks_per_node: ['8'] - n_nodes: '{scaling_nodes}' - n_gpus: 64 experiments: - scaling_{n_nodes}nodes_{size_name}: - variants: - package_manager: spack + lammps_hns-reaxff_single_node_rocm_{x}_{y}_{z}_{n_nodes}_{n_ranks_per_node}_{n_gpus}: + exclude: {} + matrix: [] variables: - size_name: ['bigbig'] - size_x: [20] - size_y: [40] - size_z: [32] - timesteps: 100 input_file: '{input_path}/in.reaxc.hns' - lammps_flags: '-k on g 1 -sf kk -pk kokkos gpu/aware off neigh half comm device neigh/qeq full newton on -v x {size_x} -v y {size_y} -v z {size_z} -nocite' - zips: - problems: - - size_name - - size_x - - size_y - - size_z - matrix: - - problems - - scaling_nodes - + lammps_flags: -v x {x} -v y {y} -v z {z} -k on g 1 -sf kk -pk kokkos + gpu/aware on neigh half comm device neigh/qeq full newton on -nocite + n_gpus: 64 + n_nodes: 8 + n_ranks_per_node: 8 + timesteps: 100 + x: 20 + y: 40 + z: 32 + variants: + package_manager: spack + zips: {} + config: + deprecated: true + spack_flags: + concretize: -U -f + install: --add --keep-stage + include: + - ./configs + modifiers: + - name: allocation software: - packages: - lammps: - pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+rocm+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff~openmp lammps_sizes=bigbig amdgpu_target={rocm_arch} - compiler: compiler-rocm environments: lammps: packages: - blas-rocm - - lapack - - mpi-rocm-no-gtl + - default-mpi - lammps + packages: + lammps: + compiler: default-compiler + pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff + lammps_sizes=bigbig ~openmp +rocm amdgpu_target={rocm_arch} diff --git a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml index 0d0287575..ba9be75db 100644 --- a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml +++ b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml @@ -21,8 +21,8 @@ compilers: modules: [] environment: prepend_path: - LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib - extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/] + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/rocm-5.5.1/lib" + extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/, /opt/rocm-5.5.1/lib] - compiler: spec: rocmcc@5.5.1 paths: diff --git a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/software.yaml b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/software.yaml index e729bc0aa..7b76fd61c 100644 --- a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/software.yaml +++ b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/software.yaml @@ -8,7 +8,7 @@ software: default-compiler: pkg_spec: cce@16.0.0-rocm5.5.1 default-mpi: - pkg_spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl + pkg_spec: cray-mpich@8.1.26%cce@16.0.0 +gtl compiler-rocm: pkg_spec: cce@16.0.0-rocm5.5.1 compiler-amdclang: diff --git a/repo/lammps/package.py b/repo/lammps/package.py index e2403af8f..230dbbafa 100644 --- a/repo/lammps/package.py +++ b/repo/lammps/package.py @@ -9,6 +9,10 @@ class Lammps(BuiltinLammps): + depends_on("kokkos+openmp", when="+openmp") + depends_on("kokkos+rocm", when="+rocm") + depends_on("kokkos+cuda", when="+cuda") + def setup_run_environment(self, env): super(BuiltinLammps, self).setup_run_environment(env) @@ -16,3 +20,11 @@ def setup_run_environment(self, env): if self.compiler.extra_rpaths: for rpath in self.compiler.extra_rpaths: env.prepend_path("LD_LIBRARY_PATH", rpath) + + def setup_build_environment(self, env): + super().setup_build_environment(env) + + spec = self.spec + if "+mpi" in spec: + if spec["mpi"].extra_attributes and "ldflags" in spec["mpi"].extra_attributes: + env.append_flags("LDFLAGS", spec["mpi"].extra_attributes["ldflags"]) diff --git a/systems/llnl-elcapitan/compilers/rocm/00-rocm-551-compilers.yaml b/systems/llnl-elcapitan/compilers/rocm/00-rocm-551-compilers.yaml index 9185da69e..6fe5e6bf3 100644 --- a/systems/llnl-elcapitan/compilers/rocm/00-rocm-551-compilers.yaml +++ b/systems/llnl-elcapitan/compilers/rocm/00-rocm-551-compilers.yaml @@ -21,8 +21,8 @@ compilers: modules: [] environment: prepend_path: - LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib - extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/] + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/rocm-5.5.1/lib" + extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/, /opt/rocm-5.5.1/lib] - compiler: spec: rocmcc@5.5.1 paths: