lammps experiment.py (#446)

* lammps experiment.py

* lint formatiing

* lint formatiing

* Turn off gpu aware mpi for rocm

* Enable gtl in lammps

* Change lammps legacy ramble configs

* Add lammps dry-runs

* undo to merge in develop

* reapply path changes

---------

Co-authored-by: Riyaz Haque <[email protected]>
Co-authored-by: pearce8 <[email protected]>

.github/workflows/run.yml

@@ -243,7 +243,17 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
-      - name: Dry run lammps/rocm on Tioga with allocation modifier
+      - name: Dry run lammps/openmp with static Ruby
+        run: |
+          ./bin/benchpark setup lammps/openmp LLNL-Ruby-icelake-OmniPath workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir workspace/lammps/openmp/LLNL-Ruby-icelake-OmniPath/workspace \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run lammps/rocm with static Tioga
         run: |
           ./bin/benchpark setup lammps/rocm LLNL-Tioga-HPECray-zen3-MI250X-Slingshot workspace/
           . workspace/setup.sh
@@ -253,6 +263,40 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
+      - name: Dry run dynamic lammps/openmp on static Ruby
+        run: |
+          ./bin/benchpark experiment init --dest=lammps-openmp lammps+openmp
+          ./bin/benchpark setup ./lammps-openmp LLNL-Ruby-icelake-OmniPath workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir workspace/lammps-openmp/LLNL-Ruby-icelake-OmniPath/workspace \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic lammps/rocm on static Tioga 
+        run: |
+          ./bin/benchpark experiment init --dest=lammps-rocm lammps+rocm
+          ./bin/benchpark setup ./lammps-rocm LLNL-Tioga-HPECray-zen3-MI250X-Slingshot workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir workspace/lammps-rocm/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/workspace \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic lammps/rocm with dynamic Tioga
+        run: |
+          ./bin/benchpark experiment init --dest=lammps-rocm-tioga lammps+rocm
+          ./bin/benchpark setup lammps-rocm-tioga ./tioga-system workspace/
+          system_id=$(./bin/benchpark system id ./tioga-system)
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir workspace/lammps-rocm-tioga/$system_id/workspace \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
       - name: Dry run hpl/openmp with Caliper-topdown modifier on Magma
         run: |
           ./bin/benchpark setup hpl/openmp --modifier=caliper-topdown LLNL-Magma-Penguin-icelake-OmniPath workspace/

experiments/lammps/experiment.py

@@ -0,0 +1,90 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.openmp import OpenMPExperiment
+from benchpark.rocm import ROCmExperiment
+
+
+class Lammps(
+    Experiment,
+    OpenMPExperiment,
+    ROCmExperiment,
+):
+    variant(
+        "workload",
+        default="hns-reaxff",
+        values=("hns-reaxff", "lj", "eam", "chain", "chute", "rhodo"),
+        description="workloads",
+    )
+
+    variant(
+        "version",
+        default="20231121",
+        description="app version",
+    )
+
+    def compute_applications_section(self):
+        if self.spec.satisfies("+openmp"):
+            problem_sizes = {"x": 8, "y": 8, "z": 8}
+            kokkos_mode = "t {n_threads_per_proc}"
+            kokkos_gpu_aware = "off"
+            kokkos_comm = "host"
+        elif self.spec.satisfies("+rocm"):
+            problem_sizes = {"x": 20, "y": 40, "z": 32}
+            kokkos_mode = "g 1"
+            kokkos_gpu_aware = "on"
+            kokkos_comm = "device"
+
+        for nk, nv in problem_sizes.items():
+            self.add_experiment_variable(nk, nv, True)
+
+        input_sizes = " ".join(f"-v {k} {{{k}}}" for k in problem_sizes.keys())
+
+        if self.spec.satisfies("+openmp"):
+            self.add_experiment_variable("n_nodes", 1, True)
+            self.add_experiment_variable("n_ranks_per_node", 36, True)
+            self.add_experiment_variable("n_threads_per_proc", 1, True)
+        elif self.spec.satisfies("+rocm"):
+            self.add_experiment_variable("n_nodes", 8, True)
+            self.add_experiment_variable("n_ranks_per_node", 8, True)
+            self.add_experiment_variable("n_gpus", 64, True)
+
+        self.add_experiment_variable("timesteps", 100, False)
+        self.add_experiment_variable("input_file", "{input_path}/in.reaxc.hns", False)
+        self.add_experiment_variable(
+            "lammps_flags",
+            f"{input_sizes} -k on {kokkos_mode} -sf kk -pk kokkos gpu/aware {kokkos_gpu_aware} neigh half comm {kokkos_comm} neigh/qeq full newton on -nocite",
+            False,
+        )
+
+    def compute_spack_section(self):
+        # get package version
+        app_version = self.spec.variants["version"][0]
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+        if self.spec.satisfies("+rocm"):
+            system_specs["rocm_arch"] = "{rocm_arch}"
+            system_specs["blas"] = "blas-rocm"
+
+        # set package spack specs
+        if self.spec.satisfies("+rocm"):
+            # empty package_specs value implies external package
+            self.add_spack_spec(system_specs["blas"])
+        # empty package_specs value implies external package
+        self.add_spack_spec(system_specs["mpi"])
+
+        self.add_spack_spec(
+            self.name,
+            [
+                f"lammps@{app_version} +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff lammps_sizes=bigbig ",
+                system_specs["compiler"],
+            ],
+        )

legacy/experiments/lammps/openmp/ramble.yaml

@@ -1,59 +1,44 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
-# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: Apache-2.0
-
 ramble:
-  include:
-  - ./configs/software.yaml
-  - ./configs/variables.yaml
-
-  config:
-    deprecated: true
-    spack_flags:
-      install: '--add --keep-stage'
-      concretize: '-U -f'
-
-  modifiers:
-  - name: allocation
-
   applications:
     lammps:
       workloads:
         hns-reaxff:
-          variables:
-            size_name: ['medium']
-            size_x: [2]
-            size_y: [2]
-            size_z: [2]
-            scaling_nodes: [1]
-            n_nodes: '{scaling_nodes}'
-            n_threads_per_proc: '1'
-            lammps_flags: '-v x {size_x} -v y {size_y} -v z {size_z}'
           experiments:
-            scaling_{n_nodes}nodes_{size_name}:
+            lammps_hns-reaxff_single_node_openmp_{x}_{y}_{z}_{n_nodes}_{n_ranks_per_node}_{n_threads_per_proc}:
+              exclude: {}
+              matrix: []
+              variables:
+                input_file: '{input_path}/in.reaxc.hns'
+                lammps_flags: -v x {x} -v y {y} -v z {z} -k on t {n_threads_per_proc}
+                  -sf kk -pk kokkos gpu/aware off neigh half comm host neigh/qeq full
+                  newton on -nocite
+                n_nodes: 1
+                n_ranks_per_node: 36
+                n_threads_per_proc: 1
+                timesteps: 100
+                x: 8
+                y: 8
+                z: 8
               variants:
                 package_manager: spack
-              variables:
-                env_name: lammps
-                n_ranks_per_node: ['36']
-              zips:
-                problems:
-                - size_name
-                - size_x
-                - size_y
-                - size_z
-              matrix:
-              - problems
-              - scaling_nodes
+              zips: {}
+  config:
+    deprecated: true
+    spack_flags:
+      concretize: -U -f
+      install: --add --keep-stage
+  include:
+  - ./configs
+  modifiers:
+  - name: allocation
   software:
-    packages:
-      lammps:
-        pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+openmp+openmp-package+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff
-        compiler: default-compiler
     environments:
       lammps:
         packages:
-        - lapack
         - default-mpi
         - lammps
+    packages:
+      lammps:
+        compiler: default-compiler
+        pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff
+          lammps_sizes=bigbig +openmp ~rocm

legacy/experiments/lammps/rocm/ramble.yaml

@@ -1,63 +1,44 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
-# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: Apache-2.0
-
 ramble:
-  include:
-  - ./configs/software.yaml
-  - ./configs/variables.yaml
-  - ./configs/modifier.yaml
-
-  config:
-    deprecated: true
-    spack_flags:
-      install: '--add --keep-stage'
-      concretize: '-U -f'
-
-  modifiers:
-  - name: allocation
-
   applications:
     lammps:
       workloads:
         hns-reaxff:
-          variables:
-            scaling_nodes: ['8']
-            n_ranks_per_node: ['8']
-            n_nodes: '{scaling_nodes}'
-            n_gpus: 64
           experiments:
-            scaling_{n_nodes}nodes_{size_name}:
-              variants:
-                package_manager: spack
+            lammps_hns-reaxff_single_node_rocm_{x}_{y}_{z}_{n_nodes}_{n_ranks_per_node}_{n_gpus}:
+              exclude: {}
+              matrix: []
               variables:
-                size_name: ['bigbig']
-                size_x: [20]
-                size_y: [40]
-                size_z: [32]
-                timesteps: 100
                 input_file: '{input_path}/in.reaxc.hns'
-                lammps_flags: '-k on g 1 -sf kk -pk kokkos gpu/aware off neigh half comm device neigh/qeq full newton on -v x {size_x} -v y {size_y} -v z {size_z} -nocite'
-              zips:
-                problems:
-                - size_name
-                - size_x
-                - size_y
-                - size_z
-              matrix:
-              - problems
-              - scaling_nodes
-
+                lammps_flags: -v x {x} -v y {y} -v z {z} -k on g 1 -sf kk -pk kokkos
+                  gpu/aware on neigh half comm device neigh/qeq full newton on -nocite
+                n_gpus: 64
+                n_nodes: 8
+                n_ranks_per_node: 8
+                timesteps: 100
+                x: 20
+                y: 40
+                z: 32
+              variants:
+                package_manager: spack
+              zips: {}
+  config:
+    deprecated: true
+    spack_flags:
+      concretize: -U -f
+      install: --add --keep-stage
+  include:
+  - ./configs
+  modifiers:
+  - name: allocation
   software:
-    packages:
-      lammps:
-        pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+rocm+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff~openmp lammps_sizes=bigbig amdgpu_target={rocm_arch}
-        compiler: compiler-rocm
     environments:
       lammps:
         packages:
         - blas-rocm
-        - lapack
-        - mpi-rocm-no-gtl
+        - default-mpi
         - lammps
+    packages:
+      lammps:
+        compiler: default-compiler
+        pkg_spec: lammps@20231121 +opt+manybody+molecule+kspace+rigid+kokkos+asphere+dpd-basic+dpd-meso+dpd-react+dpd-smooth+reaxff
+          lammps_sizes=bigbig ~openmp +rocm amdgpu_target={rocm_arch}

legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml

@@ -21,8 +21,8 @@ compilers:
     modules: []
     environment:
       prepend_path:
-        LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib
-    extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/]
+        LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/rocm-5.5.1/lib"
+    extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/, /opt/rocm-5.5.1/lib]
 - compiler:
     spec: [email protected]
     paths:

legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/software.yaml

@@ -8,7 +8,7 @@ software:
     default-compiler:
       pkg_spec: [email protected]
     default-mpi:
-      pkg_spec: [email protected]%[email protected] ~gtl
+      pkg_spec: [email protected]%[email protected] +gtl
     compiler-rocm:
       pkg_spec: [email protected]
     compiler-amdclang:

repo/lammps/package.py

@@ -9,10 +9,22 @@
 
 class Lammps(BuiltinLammps):
 
+  depends_on("kokkos+openmp", when="+openmp")
+  depends_on("kokkos+rocm", when="+rocm")
+  depends_on("kokkos+cuda", when="+cuda")
+
   def setup_run_environment(self, env):
 
     super(BuiltinLammps, self).setup_run_environment(env)
 
     if self.compiler.extra_rpaths:
       for rpath in self.compiler.extra_rpaths:
         env.prepend_path("LD_LIBRARY_PATH", rpath)
+
+  def setup_build_environment(self, env):
+    super().setup_build_environment(env)
+
+    spec = self.spec
+    if "+mpi" in spec:
+      if spec["mpi"].extra_attributes and "ldflags" in spec["mpi"].extra_attributes:
+        env.append_flags("LDFLAGS", spec["mpi"].extra_attributes["ldflags"])

systems/llnl-elcapitan/compilers/rocm/00-rocm-551-compilers.yaml

@@ -21,8 +21,8 @@ compilers:
     modules: []
     environment:
       prepend_path:
-        LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib
-    extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/]
+        LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/rocm-5.5.1/lib"
+    extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/, /opt/rocm-5.5.1/lib]
 - compiler:
     spec: [email protected]
     paths: