Skip to content

Commit

Permalink
Loki IDEM and SCC working with NEW scheduler, STACK variants not (yet)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaelSt98
MichaelSt98 committed Feb 7, 2024
1 parent 8442fc4 commit c8ea49b
Show file tree
Hide file tree
Showing 4 changed files with 222 additions and 132 deletions.
206 changes: 129 additions & 77 deletions src/ecwam/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,14 +472,27 @@ if( HAVE_WAM_LOKI )
list(APPEND loki_idem_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE idem FRONTEND ${LOKI_FRONTEND} CPP
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
OUTPUT ${loki_idem_srcs}
DEPENDS ${phys_srcs}
# loki_transform_convert(
# MODE idem FRONTEND ${LOKI_FRONTEND} CPP
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
# OUTPUT ${loki_idem_srcs}
# DEPENDS ${phys_srcs}
# )

loki_transform(
COMMAND convert
OUTPUT ${loki_idem_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
DEPENDS ${phys_srcs}
MODE idem
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand Down Expand Up @@ -511,19 +524,32 @@ if( HAVE_WAM_LOKI )
## * Internal "do-nothing" mode for Loki debug ##
############################################################
foreach(src ${phys_srcs} wamintgr.F90)
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
list(APPEND loki_idem_stack_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE idem-stack FRONTEND ${LOKI_FRONTEND} CPP
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
OUTPUT ${loki_idem_stack_srcs}
DEPENDS ${phys_srcs}
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
list(APPEND loki_idem_stack_srcs ${fnm})
endforeach()

# loki_transform_convert(
# MODE idem-stack FRONTEND ${LOKI_FRONTEND} CPP
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
# OUTPUT ${loki_idem_stack_srcs}
# DEPENDS ${phys_srcs}
# )

loki_transform(
COMMAND convert
OUTPUT ${loki_idem_stack_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
DEPENDS ${phys_srcs}
MODE idem-stack
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
CPP
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand All @@ -547,9 +573,9 @@ if( HAVE_WAM_LOKI )
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_idem_stack
INSTALL_DIRECTORY module/${ecwam}_idem_stack
)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_idem_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_idem_stack PUBLIC "-Mcray=pointer" )
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
Expand All @@ -566,14 +592,27 @@ if( HAVE_WAM_LOKI )
list(APPEND loki_scc_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE scc FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
OUTPUT ${loki_scc_srcs}
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# loki_transform_convert(
# MODE scc FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
#  OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
# OUTPUT ${loki_scc_srcs}
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# )

loki_transform(
COMMAND convert
OUTPUT ${loki_scc_srcs}
BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
MODE scc
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
FRONTEND ${LOKI_FRONTEND}
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
)

ecbuild_add_library(
Expand Down Expand Up @@ -608,55 +647,68 @@ if( HAVE_WAM_LOKI )
############################################################
## Loki SCC with pool allocator for temporaries: ##
############################################################
foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
string(REPLACE ".F90" "" fnc ${src})
string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
list(APPEND loki_scc_stack_srcs ${fnm})
endforeach()

loki_transform_convert(
MODE scc-stack FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
PATH ${CMAKE_CURRENT_SOURCE_DIR}
INCLUDES ${ecwam_intfb_includes}
OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
OUTPUT ${loki_scc_stack_srcs}
DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
)

ecbuild_add_library(
TARGET ${ecwam}_scc_stack
TYPE ${LIBRARY_TYPE}
DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
${MPI_Fortran_LIBRARIES}
${${PNAME}_OCEANMODEL_LIBRARIES}
$<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
)
#foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
# string(REPLACE ".F90" "" fnc ${src})
# string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
# list(APPEND loki_scc_stack_srcs ${fnm})
# endforeach()

# loki_transform_convert(
# MODE scc-stack FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# PATH ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
# OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
# OUTPUT ${loki_scc_stack_srcs}
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# )

#loki_transform(
# COMMAND convert
# OUTPUT ${loki_scc_stack_srcs}
# BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
# DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
# MODE scc-stack
# CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
# CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
# FRONTEND ${LOKI_FRONTEND}
# SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
# INCLUDES ${ecwam_intfb_includes}
#)

#ecbuild_add_library(
# TARGET ${ecwam}_scc_stack
# TYPE ${LIBRARY_TYPE}
# DEFINITIONS ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
# SOURCES ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
# PUBLIC_LIBS fiat parkind_${prec} ${ecwam}_intfb
# ${MPI_Fortran_LIBRARIES}
# ${${PNAME}_OCEANMODEL_LIBRARIES}
# $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
# PRIVATE_LIBS eccodes_f90
# ${MULTIO_LIBRARIES}
# ${OpenMP_Fortran_LIBRARIES}
# $<${HAVE_FIELD_API}:field_api_${prec}>
# PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
# PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
#)

ecwam_target_fortran_module_directory(
TARGET ${ecwam}_scc_stack
MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
INSTALL_DIRECTORY module/${ecwam}_scc_stack
)
#ecwam_target_fortran_module_directory(
# TARGET ${ecwam}_scc_stack
# MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
# INSTALL_DIRECTORY module/${ecwam}_scc_stack
#)

ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )
#ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )

if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
if( HAVE_ACC )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
endif()
elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
endif()
#if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
# if( HAVE_ACC )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
# endif()
#elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
# target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
#endif()
endif()
endif()

Expand Down
60 changes: 40 additions & 20 deletions src/ecwam/ecwam_loki.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,55 @@ mode = "idem"
role = "kernel"
expand = true
strict = true
disable = [
"DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim",
"jwrb", "jwru", "jwro", "environment", "frequency",
"forcing_fields", "intgt_param_fields", "wave2ocean", "abort1",
"iu06"
]
# Ensure that we are never adding these to the tree, and thus
# do not attempt to look up the source files for these.
#disable = [
# "DR_HOOK", "lhook", "jphook", "omp_get_wtime", "flush", "jwim",
# "jwrb", "jwru", "jwro", "environment", "frequency",
# "forcing_fields", "intgt_param_fields", "wave2ocean", "abort1",
# "iu06"
#]
# , 'dr_hook', 'lhook', 'jphook', '

disable = ['yomhook', 'abort', 'wave2ocean', 'intgt_param_fields',
'forcing_fields', 'environment', 'yomgstats', 'gstats', 'gstats_barrier', 'frequency_field',
'environment_field', 'source_contribs_field', 'intgt_param_fields',
'source_contribs_field', 'system_clock', 'ec_parkind', 'parkind_wave', 'parkind1',
'wam_multio_mod', 'flush', 'yowabort', 'mpl_module', 'yowtest', 'oml_mod',
'ieee_arithmetic', 'outwspec_io_serv_handler', 'outint_io_serv_handler',
'ifstowam_handler', 'mfeb_length', 'mon', 'isamesign', 'cdm', 'abor1', 'abort1', 'abort1_mod',
'yowfield_mod', 'ctuw_mod', 'wam_user_clock',
'propag_wam', 'newwind', 'incdate']

# Prune the tree for these to ensure they are not processed by transformations
block = ['ec_parkind', 'parkind_wave', 'parkind1', 'mfeb_length', 'mon', 'isamesign'] # , 'yoecldp', 'fc*_mod']

utility_routines = ['dr_hook', 'abort1', 'write(iu06']
enable_imports = false

# Define entry point for call-tree transformation
[routines]

# [routines.meansqs_gc]
# disable = ['omegagc']

[routines.wamintgr]
role = "driver"
expand = true
disable = [
"PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT",
"ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT",
"WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT",
"FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW",
"FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW",
"INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW",
"SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW",
"INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock",
"FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL",
"FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL",
"INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT",
"SOURCE_CONTRIBS_FIELD%FINAL"
]
# disable = [
# "PROPAG_WAM", "NEWWIND", "GSTATS", "FREQUENCY_FIELD%INIT",
# "ENVIRONMENT_FIELD%INIT", "FORCING_FIELDS_FIELD%INIT",
# "WAVE2OCEAN_FIELD%INIT", "INTGT_PARAM_FIELDS_FIELD%INIT",
# "FREQUENCY_FIELD%UPDATE_VIEW", "ENVIRONMENT_FIELD%UPDATE_VIEW",
# "FORCING_FIELDS_FIELD%UPDATE_VIEW", "WAVE2OCEAN_FIELD%UPDATE_VIEW",
# "INTGT_PARAM_FIELDS_FIELD%UPDATE_VIEW",
# "SOURCE_CONTRIBS_FIELD%INIT", "SOURCE_CONTRIBS_FIELD%UPDATE_VIEW",
# "INCDATE", "DR_HOOK", "omp_get_wtime", "abort1", "wam_user_clock",
# "FREQUENCY_FIELD%FINAL", "ENVIRONMENT_FIELD%FINAL",
# "FORCING_FIELDS_FIELD%FINAL", "WAVE2OCEAN_FIELD%FINAL",
# "INTGT_PARAM_FIELDS_FIELD%FINAL", "SOURCE_CONTRIBS_FIELD%INIT",
#  "SOURCE_CONTRIBS_FIELD%FINAL"
# ]
real_kind = 'JWRB'

# Define indices and bounds for array dimensions
Expand Down
Loading

0 comments on commit c8ea49b

Please sign in to comment.