OpenACC port of wave propagation (ACCV6)

.github/tools/install-intel-oneapi.sh

@@ -1,14 +1,14 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+version=2023.2.0
+KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 wget https://apt.repos.intel.com/intel-gpg-keys/$KEY
 sudo apt-key add $KEY
 rm $KEY
 echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 sudo apt-get update
 sudo apt-get install \
-    intel-oneapi-compiler-fortran \
-    intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic \
-    intel-oneapi-mpi \
-    intel-oneapi-mpi-devel \
-    intel-oneapi-mkl
+    intel-oneapi-compiler-fortran-$version \
+    intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-$version \
+    intel-oneapi-mpi-devel-2021.10.0 \
+    intel-oneapi-mkl-$version

.github/workflows/build.yml

@@ -34,8 +34,7 @@ jobs:
         name:
           - linux gnu-10
           - linux clang-12
-          - linux nvhpc-21.9
-          - linux intel
+          - linux intel-classic
           - macos
 
         include:
@@ -74,9 +73,9 @@ jobs:
             cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177
             caching: true
 
-          - name : linux intel
+          - name : linux intel-classic
             os: ubuntu-20.04
-            compiler: intel-oneapi
+            compiler: intel-classic
             compiler_cc: icc
             compiler_cxx: icpc
             compiler_fc: ifort
@@ -152,7 +151,7 @@ jobs:
         ${ECWAM_TOOLS}/install-intel-oneapi.sh
         source /opt/intel/oneapi/setvars.sh
         printenv >> $GITHUB_ENV
-        echo "CACHE_SUFFIX=$(icc -dumpversion)" >> $GITHUB_ENV
+        echo "CACHE_SUFFIX=$CC-$($CC -dumpversion)" >> $GITHUB_ENV
 
     - name: Install MPI
       shell: bash -eux {0}

CMakeLists.txt

@@ -116,6 +116,12 @@ ecbuild_add_option( FEATURE WAM_GPU
                     DESCRIPTION "Offload ecWam physics to the GPU"
                     CONDITION HAVE_FIELD_API )
 
+### CUDA-aware MPI
+ecbuild_add_option( FEATURE GPU_AWARE_MPI
+                    DEFAULT OFF
+                    REQUIRED_PACKAGES "MPI COMPONENTS Fortran"
+                    DESCRIPTION "Enable CUDA-aware MPI" )
+
 ### check for CUDA
 include(CheckLanguage)
 check_language(CUDA)

src/ecwam/CMakeLists.txt

@@ -410,6 +410,10 @@ else()
   unset( ${PNAME}_OCEANMODEL_INCLUDE_DIRS )
 endif()
 
+if( HAVE_GPU_AWARE_MPI )
+  list(APPEND ECWAM_DEFINITIONS WITH_GPU_AWARE_MPI )
+endif()
+
 # Using dynamic linking creates undefined references to the device
 # copies of module global variables
 set( LIBRARY_TYPE SHARED )
@@ -432,6 +436,7 @@ ecbuild_add_library(
                      ${MULTIO_LIBRARIES}
                      ${OpenMP_Fortran_LIBRARIES}
                      $<${HAVE_FIELD_API}:field_api_${prec}>
+		     MPI::MPI_Fortran
     PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
     PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS}
 )
@@ -493,6 +498,7 @@ if( HAVE_WAM_LOKI )
                        ${MULTIO_LIBRARIES}
                        ${OpenMP_Fortran_LIBRARIES}
                        $<${HAVE_FIELD_API}:field_api_${prec}>
+		       MPI::MPI_Fortran
       PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
       PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
   )
@@ -537,6 +543,7 @@ if( HAVE_WAM_LOKI )
                        ${MULTIO_LIBRARIES}
                        ${OpenMP_Fortran_LIBRARIES}
                        $<${HAVE_FIELD_API}:field_api_${prec}>
+		       MPI::MPI_Fortran
       PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
       PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
   )
@@ -588,6 +595,7 @@ if( HAVE_WAM_LOKI )
      					 ${MULTIO_LIBRARIES}
      					 ${OpenMP_Fortran_LIBRARIES}
      					 $<${HAVE_FIELD_API}:field_api_${prec}>
+					 MPI::MPI_Fortran
      	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
      	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
      )
@@ -636,6 +644,7 @@ if( HAVE_WAM_LOKI )
      					 ${MULTIO_LIBRARIES}
      					 ${OpenMP_Fortran_LIBRARIES}
      					 $<${HAVE_FIELD_API}:field_api_${prec}>
+					 MPI::MPI_Fortran
      	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
      	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
      )
@@ -684,6 +693,7 @@ if( HAVE_CUDA )
   					 ${MULTIO_LIBRARIES}
   					 ${OpenMP_Fortran_LIBRARIES}
   					 $<${HAVE_FIELD_API}:field_api_${prec}>
+					 MPI::MPI_Fortran
   	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
   	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
   )