Skip to content

Commit

Permalink
Re-enable SCC-Stack for compatibility with new scheduler
Browse files Browse the repository at this point in the history
  • Loading branch information
@reuterbal
reuterbal committed Feb 12, 2024
1 parent c8ea49b commit 8bc628f
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 177 deletions.
302 changes: 131 additions & 171 deletions src/ecwam/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,32 +467,22 @@ if( HAVE_WAM_LOKI )
## * Internal "do-nothing" mode for Loki debug ##
############################################################
foreach(src ${phys_srcs} wamintgr.F90)
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
list(APPEND loki_idem_srcs ${fnm})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
list(APPEND loki_idem_srcs ${fnm})
endforeach()

# loki_transform_convert(
# MODE idem FRONTEND ${LOKI_FRONTEND} CPP
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
# OUTPUT ${loki_idem_srcs}
# DEPENDS ${phys_srcs}
# )

loki_transform(
COMMAND convert
OUTPUT ${loki_idem_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
DEPENDS ${phys_srcs}
MODE idem
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
COMMAND convert
OUTPUT ${loki_idem_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
DEPENDS ${phys_srcs}
MODE idem
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand Down Expand Up @@ -527,29 +517,19 @@ if( HAVE_WAM_LOKI )
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
list(APPEND loki_idem_stack_srcs ${fnm})
endforeach()

# loki_transform_convert(
# MODE idem-stack FRONTEND ${LOKI_FRONTEND} CPP
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
# OUTPUT ${loki_idem_stack_srcs}
# DEPENDS ${phys_srcs}
# )
endforeach()

loki_transform(
COMMAND convert
OUTPUT ${loki_idem_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
DEPENDS ${phys_srcs}
MODE idem-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
COMMAND convert
OUTPUT ${loki_idem_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
DEPENDS ${phys_srcs}
MODE idem-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand All @@ -573,9 +553,9 @@ if( HAVE_WAM_LOKI )
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_idem_stack
INSTALL_DIRECTORY module/${ecwam}_idem_stack
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_idem_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_idem_stack PUBLIC "-Mcray=pointer" )
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
Expand All @@ -586,23 +566,13 @@ if( HAVE_WAM_LOKI )
############################################################
## Loki SCC transformation: ##
############################################################
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
list(APPEND loki_scc_srcs ${fnm})
endforeach()

# loki_transform_convert(
# MODE scc FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
#  OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
# OUTPUT ${loki_scc_srcs}
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# )

loki_transform(
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
list(APPEND loki_scc_srcs ${fnm})
endforeach()

loki_transform(
COMMAND convert
OUTPUT ${loki_scc_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
Expand All @@ -613,102 +583,92 @@ if( HAVE_WAM_LOKI )
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
TARGET ${ecwam}_scc
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
)

ecbuild_add_library(
TARGET ${ecwam}_scc
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)
ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
INSTALL_DIRECTORY module/${ecwam}_scc
)
ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc )
if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND HAVE_ACC )
target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
INSTALL_DIRECTORY module/${ecwam}_scc
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND HAVE_ACC )
target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()

############################################################
## Loki SCC with pool allocator for temporaries: ##
############################################################
#foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
# string(REPLACE ".F90" "" fnc ${src})
# string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
# list(APPEND loki_scc_stack_srcs ${fnm})
# endforeach()

# loki_transform_convert(
# MODE scc-stack FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
# OUTPUT ${loki_scc_stack_srcs}
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# )

#loki_transform(
# COMMAND convert
# OUTPUT ${loki_scc_stack_srcs}
# BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# MODE scc-stack
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# FRONTEND ${LOKI_FRONTEND}
# SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
#)

#ecbuild_add_library(
# TARGET ${ecwam}_scc_stack
# TYPE ${LIBRARY_TYPE}
# DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
# SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
# PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
# ${MPI_Fortran_LIBRARIES}
# ${${PNAME}_OCEANMODEL_LIBRARIES}
# $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
# PRIVATE_LIBS eccodes_f90
# ${MULTIO_LIBRARIES}
# ${OpenMP_Fortran_LIBRARIES}
# $<${HAVE_FIELD_API}:field_api_${prec}>
# PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
# PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
#)

#ecwam_target_fortran_module_directory(
# TARGET ${ecwam}_scc_stack
# MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
# INSTALL_DIRECTORY module/${ecwam}_scc_stack
#)

#ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )

#if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
# if( HAVE_ACC )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
# endif()
#elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
#endif()
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
list(APPEND loki_scc_stack_srcs ${fnm})
endforeach()

loki_transform(
COMMAND convert
OUTPUT ${loki_scc_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
MODE scc-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
TARGET ${ecwam}_scc_stack
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_stack
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
INSTALL_DIRECTORY module/${ecwam}_scc_stack
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
if( HAVE_ACC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
endif()
endif()
endif()

Expand All @@ -725,28 +685,28 @@ if( HAVE_CUDA )
set_source_files_properties( ${wam_scc_cuf_srcs} PROPERTIES COMPILE_OPTIONS "-Mcuda=maxregcount:128" )

ecbuild_add_library(
TARGET ${ecwam}_scc_cuf
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_CUDA
SOURCES ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
TARGET ${ecwam}_scc_cuf
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_CUDA
SOURCES ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_cuf
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
INSTALL_DIRECTORY module/${ecwam}_scc_cuf
TARGET ${ecwam}_scc_cuf
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
INSTALL_DIRECTORY module/${ecwam}_scc_cuf
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_cuf )

target_link_options( ${ecwam}_scc_cuf PUBLIC "-cuda;-gpu=pinned" )
Expand Down
12 changes: 6 additions & 6 deletions src/programs/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ if( HAVE_WAM_LOKI )
)
ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc)

# ecbuild_add_executable(
# TARGET ${PROJECT_NAME}-chief-loki-scc-stack
# SOURCES chief.F90
# LIBS ${PROJECT_NAME}_scc_stack ${OpenMP_Fortran_LIBRARIES}
# )
# ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc-stack)
ecbuild_add_executable(
TARGET ${PROJECT_NAME}-chief-loki-scc-stack
SOURCES chief.F90
LIBS ${PROJECT_NAME}_scc_stack ${OpenMP_Fortran_LIBRARIES}
)
ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc-stack)
endif()
endif()

Expand Down

0 comments on commit 8bc628f

Please sign in to comment.