gromacs (#197)

* Create gromacs/application.py * Add gromacs openmp config (#62) * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll <[email protected]> * Update repo/gromacs/application.py Co-authored-by: Szilárd Páll <[email protected]> * copyright * copyright * copyright * Draft ADAC experiment setup * CUDA experiment for LLNL-Sierra-IBM-power9-V100-Infiniband * CUDA config for LLNL-Pascal-Penguin-broadwell-P100-OmniPat * Fix target name * CUDA config for lassen * Fix cuda config * Set Openmp thread affinity to cores * Fix cuda compiler for gromacs * gromacs rocm config * gromacs rocm config * Remove spack_setup * ROCm packages: externals consistency (#150) * remove version locking from experiment config; achieve this with a mixin package * add needed externals, proper dependency reference for mixin * also align rocmcc version w/hip version * Spack package from Loic Pottier * Remove duplicate config entry * Fix gromacs configs * gromacs config changes * Fix gromacs openmp config * Fix license --------- Co-authored-by: pearce8 <[email protected]> Co-authored-by: Szilárd Páll <[email protected]> Co-authored-by: Riyaz Haque <[email protected]> Co-authored-by: Peter Scheibel <[email protected]>
LLNL · Jun 21, 2024 · 0dd18bd · 0dd18bd
1 parent 9d83318
commit 0dd18bd
Show file tree

Hide file tree

Showing 8 changed files with 270 additions and 86 deletions.
diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/compilers.yaml
@@ -23,21 +23,55 @@ compilers:
         LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib
     extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/]
 - compiler:
-    spec: [email protected].5.1
+    spec: rocmcc@5.5.1
     paths:
       cc:  /opt/rocm-5.5.1/bin/amdclang
       cxx:  /opt/rocm-5.5.1/bin/amdclang++
       f77: /opt/rocm-5.5.1/bin/amdflang 
       fc:  /opt/rocm-5.5.1/bin/amdflang
     flags:
-      cflags: -g -O2 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr
-      cxxflags: -g -O2 -std=c++17 --gcc-toolchain=/opt/rh/gcc-toolset-12/root/usr
-      fflags: -g -O2 -hnopattern
+      cflags: -g -O2
+      cxxflags: -g -O2
     operating_system: rhel8
     target: x86_64
     modules: []
-    environment: {}
-    extra_rpaths: []
+    environment:
+      set:
+        RFE_811452_DISABLE: '1'
+      append_path:
+        LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs
+      prepend_path:
+        LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib"
+        LIBRARY_PATH: /opt/rocm-5.5.1/lib
+    extra_rpaths:
+    - /opt/rocm-5.5.1/lib
+    - /opt/cray/pe/gcc-libs
+    - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib
+- compiler:
+    spec: [email protected]
+    paths:
+      cc:  /opt/rocm-5.4.3/bin/amdclang
+      cxx:  /opt/rocm-5.4.3/bin/amdclang++
+      f77: /opt/rocm-5.4.3/bin/amdflang 
+      fc:  /opt/rocm-5.4.3/bin/amdflang
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment:
+      set:
+        RFE_811452_DISABLE: '1'
+      append_path:
+        LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs
+      prepend_path:
+        LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib"
+        LIBRARY_PATH: /opt/rocm-5.4.3/lib
+    extra_rpaths:
+    - /opt/rocm-5.4.3/lib
+    - /opt/cray/pe/gcc-libs
+    - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib
 - compiler:
     spec: [email protected]
     paths:

diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml
@@ -183,6 +183,55 @@ packages:
     - spec: [email protected]
       prefix: /opt/rocm-5.5.1
       buildable: false
+  comgr:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/
+  hipsparse:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1
+  hipblas:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/
+  hsakmt-roct:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/
+  roctracer-dev-api:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/
+  rocminfo:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/
+  llvm:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.4.3/llvm
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.1/llvm
   llvm-amdgpu:
     externals:
     - spec: [email protected]
@@ -215,6 +264,13 @@ packages:
       prefix: /opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/
   lapack:
     buildable: false
+  blas:
+    buildable: false
+  unzip:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr
   hypre:
     variants: amdgpu_target=gfx90a
   hwloc:
@@ -231,22 +287,27 @@ packages:
     buildable: false
   mpi:
     buildable: false
+  libfabric:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/cray/libfabric/2.1
+    buildable: false
   cray-mpich:
     buildable: false
     externals:
-    - spec: [email protected]%cce +gtl +wrappers
+    - spec: [email protected]%cce@16.0.0 +gtl +wrappers
       prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0
       extra_attributes:
         gtl_cutoff_size: 4096
         fi_cxi_ats: 0
         gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
         ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa"
-    - spec: [email protected]%cce ~gtl +wrappers
+    - spec: [email protected]%cce@16.0.0 ~gtl +wrappers
       prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0
       extra_attributes:
         gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
         ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib"
-    - spec: [email protected]%gcc ~gtl +wrappers
+    - spec: [email protected]%gcc@12.2.0 ~gtl +wrappers
       prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3
       extra_attributes:
         gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib

diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/spack.yaml
@@ -6,28 +6,28 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: cce@16
+      spack_spec: cce@16.0.0-rocm5.5.1
     default-mpi:
       spack_spec: [email protected]%[email protected] ~gtl
     compiler-rocm:
-      spack_spec: cce@16
+      spack_spec: cce@16.0.0-rocm5.5.1
     compiler-amdclang:
       spack_spec: [email protected]
     compiler-gcc:
       spack_spec: [email protected]
     blas-rocm:
-      spack_spec: [email protected]
+      spack_spec: [email protected].1
     blas:
-      spack_spec: [email protected]
+      spack_spec: [email protected].1
     lapack-rocm:
-      spack_spec: [email protected]
+      spack_spec: [email protected].1
     lapack:
       spack_spec: cray-libsci@23
     mpi-rocm-gtl:
-      spack_spec: [email protected]%cce +gtl
+      spack_spec: [email protected].26%cce@16.0.0 +gtl
     mpi-rocm-no-gtl:
       spack_spec: [email protected]%[email protected] ~gtl
     mpi-gcc:
-      spack_spec: [email protected]%gcc ~gtl
+      spack_spec: [email protected].26%gcc@12.2.0 ~gtl
     fftw:
       spack_spec: [email protected] 
diff --git a/experiments/gromacs/openmp/ramble.yaml b/experiments/gromacs/openmp/ramble.yaml
@@ -48,7 +48,7 @@ ramble:
         spack_spec: [email protected] +mpi+openmp
         compiler: default-compiler
       gromacs:
-        spack_spec: gromacs@main +mpi+openmp~hwloc
+        spack_spec: gromacs@main +mpi+openmp~hwloc~sycl~rocm~cuda
         compiler: default-compiler
     environments:
       gromacs:

diff --git a/experiments/gromacs/rocm/ramble.yaml b/experiments/gromacs/rocm/ramble.yaml
@@ -34,28 +34,28 @@ ramble:
                 pin: 'off'
                 nb: '{target}'
                 pme: 'auto'
-                bonded: '{target}'
+                bonded: 'cpu'
                 update: '{target}'
                 maxh: '0.05'
                 nsteps: '1000'
                 nstlist: '200'
-                npme: '0'
+                npme: '1'
                 size: '1536'
 
   spack:
     packages:
       hipsycl:
-        spack_spec: [email protected]
+        spack_spec: [email protected]+rocm amdgpu_target={rocm_arch}
         compiler: default-compiler
       gromacs:
-        spack_spec: gromacs@2023.3 +mpi+openmp+rocm~hwloc~double amdgpu_target={rocm_arch}
+        spack_spec: gromacs@2024 +mpi+openmp+sycl+rocm~hwloc~double amdgpu_target={rocm_arch}
         compiler: default-compiler
     environments:
       gromacs:
         packages:
-        - hipsycl
+        - default-mpi
         - blas-rocm
         - lapack-rocm
-        - mpi-rocm-gtl
+        - hipsycl
         - fftw
         - gromacs