Skip to content

Commit

Permalink
Merge pull request #12 from ecmwf-ifs/nams_naan-phys-gpu
Browse files Browse the repository at this point in the history 
 Compatibility with Loki v0.2
  • Loading branch information
@awnawab
awnawab authored Mar 22, 2024
2 parents 4654adb + 20600d9 commit 0bdd037
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 6,953 deletions.
263 changes: 135 additions & 128 deletions src/ecwam/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,10 +360,6 @@ else()
list( APPEND ecwam_srcs yowfield_mod.F90)
endif()

if( NOT HAVE_FIELD_API )
list( APPEND ecwam_srcs field_module.F90)
endif()

unset( MPI_Fortran_LIBRARIES )
if( HAVE_UNWAM )
list(APPEND ecwam_srcs ${unwam_srcs})
Expand Down Expand Up @@ -437,7 +433,7 @@ ecbuild_add_library(
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS}
)
Expand Down Expand Up @@ -472,19 +468,22 @@ if( HAVE_WAM_LOKI )
## * Internal "do-nothing" mode for Loki debug ##
############################################################
foreach(src ${phys_srcs} wamintgr.F90)
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
list(APPEND loki_idem_srcs ${fnm})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
list(APPEND loki_idem_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE idem FRONTEND ${LOKI_FRONTEND} CPP
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
loki_transform(
COMMAND convert
OUTPUT ${loki_idem_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
DEPENDS ${phys_srcs}
MODE idem
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand All @@ -499,7 +498,7 @@ if( HAVE_WAM_LOKI )
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)
Expand All @@ -517,19 +516,22 @@ if( HAVE_WAM_LOKI )
## * Internal "do-nothing" mode for Loki debug ##
############################################################
foreach(src ${phys_srcs} wamintgr.F90)
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
list(APPEND loki_idem_stack_srcs ${fnm})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
list(APPEND loki_idem_stack_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE idem-stack FRONTEND ${LOKI_FRONTEND} CPP
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
loki_transform(
COMMAND convert
OUTPUT ${loki_idem_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
DEPENDS ${phys_srcs}
MODE idem-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand All @@ -544,7 +546,7 @@ if( HAVE_WAM_LOKI )
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)
Expand All @@ -567,105 +569,110 @@ if( HAVE_WAM_LOKI )
############################################################
## Loki SCC transformation: ##
############################################################
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
list(APPEND loki_scc_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE scc FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
OUTPUT ${loki_scc_srcs}
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
)

ecbuild_add_library(
TARGET ${ecwam}_scc
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
list(APPEND loki_scc_srcs ${fnm})
endforeach()

loki_transform(
COMMAND convert
OUTPUT ${loki_scc_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
MODE scc
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
TARGET ${ecwam}_scc
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
INSTALL_DIRECTORY module/${ecwam}_scc
TARGET ${ecwam}_scc
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
INSTALL_DIRECTORY module/${ecwam}_scc
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND HAVE_ACC )
target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()

############################################################
## Loki SCC with pool allocator for temporaries: ##
############################################################
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
list(APPEND loki_scc_stack_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE scc-stack FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
OUTPUT ${loki_scc_stack_srcs}
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
)

ecbuild_add_library(
TARGET ${ecwam}_scc_stack
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_stack
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
INSTALL_DIRECTORY module/${ecwam}_scc_stack
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
if( HAVE_ACC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
endif()
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
list(APPEND loki_scc_stack_srcs ${fnm})
endforeach()

loki_transform(
COMMAND convert
OUTPUT ${loki_scc_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
MODE scc-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)
ecbuild_add_library(
TARGET ${ecwam}_scc_stack
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_stack
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
INSTALL_DIRECTORY module/${ecwam}_scc_stack
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
if( HAVE_ACC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
endif()
endif()
endif()

Expand All @@ -682,29 +689,29 @@ if( HAVE_CUDA )
set_source_files_properties( ${wam_scc_cuf_srcs} PROPERTIES COMPILE_OPTIONS "-Mcuda=maxregcount:128" )

ecbuild_add_library(
TARGET ${ecwam}_scc_cuf
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_CUDA
SOURCES ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
TARGET ${ecwam}_scc_cuf
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_CUDA
SOURCES ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
MPI::MPI_Fortran
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_cuf
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
INSTALL_DIRECTORY module/${ecwam}_scc_cuf
TARGET ${ecwam}_scc_cuf
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
INSTALL_DIRECTORY module/${ecwam}_scc_cuf
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_cuf )

target_link_options( ${ecwam}_scc_cuf PUBLIC "-cuda;-gpu=pinned" )
Expand Down
Loading

0 comments on commit 0bdd037

Please sign in to comment.