Merge pull request #29 from ecmwf-ifs/naan-promote-data-movement

Run all of `WAMINTGR` on GPU

cmake/ecwam_expand_drv_types.cmake

@@ -14,10 +14,14 @@
 
 macro( ecwam_expand_drv_types )
 
-   if( ${PNAME}_OCEANMODEL_HAVE_SINGLE_PRECISION )
+   if( ${OCEAN_PREC} STREQUAL SP )
       list(APPEND FYPP_ARGS -DPARKIND1_SINGLE_NEMO)
    endif()
 
+   if( HAVE_ACC )
+      list(APPEND FYPP_ARGS -DWAM_GPU)
+   endif()
+
    execute_process(
        COMMAND ${ECWAM_PYTHON_INTERP} -c
        "import sys; sys.path.append('${CMAKE_CURRENT_SOURCE_DIR}/../../share/ecwam/scripts'); \

src/ecwam/CMakeLists.txt

@@ -362,6 +362,7 @@ list( APPEND ecwam_srcs
 if(HAVE_LOKI AND NOT LOKI_MODE MATCHES "idem|idem-stack")
   list(APPEND ecwam_srcs wamintgr_loki_gpu.F90)
   list(REMOVE_ITEM ecwam_srcs wamintgr.F90)
+  list(APPEND ecwam_srcs cireduce_loki_gpu.F90)
 endif()
 
 # expand derived-types using src/ecwam/yowdrvtype_config.yml
@@ -404,11 +405,11 @@ endif()
 set( ${PNAME}_OCEANMODEL_LIBRARIES "" )
 if( HAVE_OCEAN_COUPLING )
   list(APPEND ECWAM_DEFINITIONS WITH_NEMO )
-  if( ${OCEAN_PREC} STREQUAL SP )
-    list(APPEND ECWAM_DEFINITIONS PARKIND1_SINGLE_NEMO )
-  endif()
   set( ${PNAME}_OCEANMODEL_LIBRARIES nemogcmcoup.${OCEAN_PREC} )
 endif()
+if( ${OCEAN_PREC} STREQUAL SP )
+  list(APPEND ECWAM_DEFINITIONS PARKIND1_SINGLE_NEMO )
+endif()
 
 if( HAVE_ECFLOW )
   list(APPEND ECWAM_PRIVATE_DEFINITIONS WAM_HAVE_ECFLOW)

src/ecwam/cireduce_loki_gpu.F90

@@ -0,0 +1,113 @@
+! (C) Copyright 1989- ECMWF.
+! 
+! This software is licensed under the terms of the Apache Licence Version 2.0
+! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+! In applying this licence, ECMWF does not waive the privileges and immunities
+! granted to it by virtue of its status as an intergovernmental organisation
+! nor does it submit to any jurisdiction.
+!
+
+SUBROUTINE CIREDUCE_LOKI_GPU (WVPRPT, FF_NOW)
+
+! ----------------------------------------------------------------------
+
+!**** *CIREDUCE* - COMPUTE SEA ICE REDUCTION FACTOR FOR SOURCE TERMS 
+!                  AND THE SEA ICE WAVE ATTENUATION FACTORS
+
+!           IF THERE IS NO SEA ICE INFORMATION OR
+!           ALL SEA ICE COVER POINTS WILL BE MASKED
+!           THEN CIWA WILL BE SET ON THE FIRST CALL. NOTHING WILL BE DONE
+!           IN ALL FOLLOWING CALLS
+
+!!!! currently also setting parametric sea ice thickness !!!!
+
+!*    PURPOSE.
+!     --------
+
+!       CIREDUCE COMPUTES SEA ICE SOURCE TERM REDUCTION FACTOR.
+
+!**   INTERFACE.
+!     ----------
+
+!       *CALL* *CIREDUCE (CGROUP, CICOVER, CITHICK, CIWA)
+
+!          *CGROUP*  - GROUP SPEED.
+!          *CICOVER* - SEA ICE COVER.
+!          *CITHICK* - SEA ICE THICKNESS. 
+!          *CIWA*-     SEA ICE WAVE ATTENUATION FACTOR. 
+
+!     METHOD.
+!     -------
+
+!     EXTERNALS.
+!     ----------
+
+
+! ----------------------------------------------------------------------
+
+      USE PARKIND_WAVE, ONLY : JWIM, JWRB, JWRU
+
+      USE YOWGRID  , ONLY : NPROMA_WAM, NCHNK
+      USE YOWICE   , ONLY : LICERUN  ,LMASKICE 
+      USE YOWPARAM , ONLY : NFRE
+
+      USE YOMHOOK  , ONLY : LHOOK,   DR_HOOK, JPHOOK
+      USE YOWDRVTYPE ,ONLY: FREQUENCY, FORCING_FIELDS
+      USE YOWSTAT, ONLY: LUPDATE_GPU_GLOBALS
+
+! ----------------------------------------------------------------------
+      IMPLICIT NONE
+
+#include "ciwaf.intfb.h"
+
+      TYPE(FREQUENCY), INTENT(INOUT)            :: WVPRPT
+      TYPE(FORCING_FIELDS), INTENT(IN)          :: FF_NOW
+
+
+      INTEGER(KIND=JWIM) :: IJ, M 
+      INTEGER(KIND=JWIM) :: ICHNK
+
+      REAL(KIND=JPHOOK) :: ZHOOK_HANDLE
+
+      LOGICAL, SAVE :: LLFRST
+
+      DATA LLFRST / .TRUE. /
+
+! ----------------------------------------------------------------------
+
+IF (LHOOK) CALL DR_HOOK('CIREDUCE',0,ZHOOK_HANDLE)
+
+        IF( .NOT. LICERUN .OR. LMASKICE ) THEN
+
+          IF (LLFRST) THEN
+            LLFRST=.FALSE.
+!           NO REDUCTION, EITHER THERE IS NO SEA ICE INFORMATION OR
+!           ALL SEA ICE COVER POINTS WILL BE MASKED
+            CALL GSTATS(1493,0)
+!$acc kernels present(WVPRPT)
+            DO ICHNK = 1, NCHNK
+               WVPRPT%CIWA(:,:,ICHNK) = 1.0_JWRB
+            ENDDO
+!$acc end kernels
+            CALL GSTATS(1493,1)
+          ENDIF
+
+        ELSE
+
+IF(LUPDATE_GPU_GLOBALS)THEN
+!$loki update_device
+ENDIF
+          CALL GSTATS(1493,0)
+!         DETERMINE THE WAVE ATTENUATION FACTOR
+!$acc parallel loop gang present(FF_NOW, WVPRPT) vector_length(NPROMA_WAM)
+          DO ICHNK = 1, NCHNK
+            CALL CIWAF(1, NPROMA_WAM, WVPRPT%CGROUP(:,:,ICHNK), FF_NOW%CICOVER(:,ICHNK), &
+&                      FF_NOW%CITHICK(:,ICHNK), WVPRPT%CIWA(:,:,ICHNK))
+          ENDDO
+!$acc end parallel loop
+          CALL GSTATS(1493,1)
+        ENDIF
+
+IF (LHOOK) CALL DR_HOOK('CIREDUCE',1,ZHOOK_HANDLE)
+
+END SUBROUTINE CIREDUCE_LOKI_GPU

src/ecwam/ciwaf.F90

@@ -60,10 +60,10 @@ SUBROUTINE CIWAF (KIJS, KIJL, CGROUP, CICOVER, CITHICK, CIWA)
       IMPLICIT NONE
 
       INTEGER(KIND=JWIM), INTENT(IN) :: KIJS, KIJL 
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL,NFRE), INTENT(IN) :: CGROUP
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL), INTENT(IN) :: CICOVER
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL), INTENT(IN) :: CITHICK
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL,NFRE), INTENT(OUT) :: CIWA
+      REAL(KIND=JWRB),DIMENSION(KIJL,NFRE), INTENT(IN) :: CGROUP
+      REAL(KIND=JWRB),DIMENSION(KIJL), INTENT(IN) :: CICOVER
+      REAL(KIND=JWRB),DIMENSION(KIJL), INTENT(IN) :: CITHICK
+      REAL(KIND=JWRB),DIMENSION(KIJL,NFRE), INTENT(OUT) :: CIWA
 
 
       INTEGER(KIND=JWIM) :: ICM, I, MAXICM
@@ -76,8 +76,8 @@ SUBROUTINE CIWAF (KIJS, KIJL, CGROUP, CICOVER, CITHICK, CIWA)
       REAL(KIND=JWRB) :: A, B, C
       REAL(KIND=JWRB) :: CIDEAC_INT, WT, WT1, WH, WH1 
       REAL(KIND=JPHOOK) :: ZHOOK_HANDLE
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL) :: DINV
-      REAL(KIND=JWRB),DIMENSION(KIJS:KIJL,NFRE) :: ALP 
+      REAL(KIND=JWRB),DIMENSION(KIJL) :: DINV
+      REAL(KIND=JWRB),DIMENSION(KIJL) :: ALP 
 
 ! ----------------------------------------------------------------------
 
@@ -137,16 +137,14 @@ SUBROUTINE CIWAF (KIJS, KIJL, CGROUP, CICOVER, CITHICK, CIWA)
             CIDEAC_INT=WT*(WH*CIDEAC(IT,IH)+ WH1*CIDEAC(IT,IH1)) +      &
      &                WT1*(WH*CIDEAC(IT1,IH)+WH1*CIDEAC(IT1,IH1))
 !!!            ALP(IJ,M)=CICOVER(IJ)*CIDEAC_INT*DINV(IJ)
-            ALP(IJ,M)=CICOVER(IJ)*EXP(CIDEAC_INT)*DINV(IJ)
+            ALP(IJ)=CICOVER(IJ)*EXP(CIDEAC_INT)*DINV(IJ)
           ELSE
-            ALP(IJ,M)=0.0_JWRB
+            ALP(IJ)=0.0_JWRB
           ENDIF
         ENDDO
-      ENDDO
 
-      DO M=1,NFRE
         DO IJ=KIJS,KIJL
-          X=ALP(IJ,M)*CGROUP(IJ,M)*IDELT
+          X=ALP(IJ)*CGROUP(IJ,M)*IDELT
           IF(X.LT.EPSMIN) THEN
             CIWA(IJ,M)=1.0_JWRB
           ELSE IF(CICOVER(IJ) > CIBLOCK) THEN

src/ecwam/ctuwupdt.F90

@@ -87,8 +87,6 @@ SUBROUTINE CTUWUPDT (IJS, IJL, NINF, NSUP,                  &
 
 IF (LHOOK) CALL DR_HOOK('CTUWUPDT',0,ZHOOK_HANDLE)
 
-!$acc update device(sinth,costh)
-!$acc update device(COSPH, nang, nfre_red)
 ! DEFINE JXO, JYO, KCR
 IF (LFRSTCTU) THEN
 
@@ -106,7 +104,7 @@ SUBROUTINE CTUWUPDT (IJS, IJL, NINF, NSUP,                  &
   IF (.NOT. ALLOCATED(JYO)) ALLOCATE(JYO(NANG,2))
   IF (.NOT. ALLOCATED(KCR)) ALLOCATE(KCR(NANG,4))
 
-!$acc update device(KLON, KLAT, KCOR, JXO, JYO, KCR, KPM)
+!$acc update device(JXO, JYO, KCR, KPM)
  !$acc kernels
   DO K=1,NANG
 
@@ -197,8 +195,6 @@ SUBROUTINE CTUWUPDT (IJS, IJL, NINF, NSUP,                  &
    NPROMA=(IJL-IJS+1)/MTHREADS + 1
 
 #ifdef _OPENACC
-!$acc update device(WLAT,WCOR)
-!$acc update device(NFRE_RED,ZPI,FR,DELTH,NANG)
 !$acc data present(KLAT,WLAT,KCOR,WCOR,WLATN,WLONN,WCORN)
 #else
 !$OMP   PARALLEL DO SCHEDULE(DYNAMIC,1) PRIVATE(JKGLO, KIJS, KIJL)
@@ -351,8 +347,6 @@ SUBROUTINE CTUWUPDT (IJS, IJL, NINF, NSUP,                  &
 !$acc end parallel
 ENDIF
 
-!$acc exit data delete(BLK2GLO)
-
 IF (ALLOCATED(THDD)) DEALLOCATE(THDD)
 IF (ALLOCATED(THDC)) DEALLOCATE(THDC)
 IF (ALLOCATED(SDOT)) DEALLOCATE(SDOT)