From 8bc628f8e5e663b27e51f0282ded92ca86fc2957 Mon Sep 17 00:00:00 2001
From: Balthasar Reuter <balthasar.reuter@ecmwf.int>
Date: Mon, 12 Feb 2024 16:00:12 +0000
Subject: [PATCH] Re-enable SCC-Stack for compatibility with new scheduler

---
 src/ecwam/CMakeLists.txt    | 302 ++++++++++++++++--------------------
 src/programs/CMakeLists.txt |  12 +-
 2 files changed, 137 insertions(+), 177 deletions(-)

diff --git a/src/ecwam/CMakeLists.txt b/src/ecwam/CMakeLists.txt
index 73205ecb..56a4879c 100644
--- a/src/ecwam/CMakeLists.txt
+++ b/src/ecwam/CMakeLists.txt
@@ -467,32 +467,22 @@ if( HAVE_WAM_LOKI )
 ##   * Internal "do-nothing" mode for Loki debug          ##
 ############################################################
   foreach(src ${phys_srcs} wamintgr.F90)
-     string(REPLACE ".F90" "" fnc ${src})
-     string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
-     list(APPEND loki_idem_srcs ${fnm})
+    string(REPLACE ".F90" "" fnc ${src})
+    string(CONCAT fnm "loki-idem/" ${fnc} ".idem.F90")
+    list(APPEND loki_idem_srcs ${fnm})
   endforeach()
 
-  # loki_transform_convert(
-  #     MODE idem FRONTEND ${LOKI_FRONTEND} CPP
-  #     CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
-  #     PATH ${CMAKE_CURRENT_SOURCE_DIR}
-  #     INCLUDES ${ecwam_intfb_includes}
-  #     OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
-  #     OUTPUT ${loki_idem_srcs}
-  #     DEPENDS ${phys_srcs}
-  # )
-
   loki_transform(
-        COMMAND convert
-        OUTPUT ${loki_idem_srcs}
-        BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
-        DEPENDS ${phys_srcs}
-        MODE idem
-        CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
-        CPP
-        FRONTEND ${LOKI_FRONTEND}
-        SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
-        INCLUDES ${ecwam_intfb_includes} 
+      COMMAND convert
+      OUTPUT ${loki_idem_srcs}
+      BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem
+      DEPENDS ${phys_srcs}
+      MODE idem
+      CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
+      CPP
+      FRONTEND ${LOKI_FRONTEND}
+      SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
+      INCLUDES ${ecwam_intfb_includes}
   )
 
   ecbuild_add_library(
@@ -527,29 +517,19 @@ if( HAVE_WAM_LOKI )
     string(REPLACE ".F90" "" fnc ${src})
     string(CONCAT fnm "loki-idem-stack/" ${fnc} ".idem_stack.F90")
     list(APPEND loki_idem_stack_srcs ${fnm})
- endforeach()
-
-  # loki_transform_convert(
-  #     MODE idem-stack FRONTEND ${LOKI_FRONTEND} CPP
-  #     CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
-  #     PATH ${CMAKE_CURRENT_SOURCE_DIR}
-  #     INCLUDES ${ecwam_intfb_includes}
-  #     OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
-  #     OUTPUT ${loki_idem_stack_srcs}
-  #     DEPENDS ${phys_srcs}
-  # )
+  endforeach()
 
   loki_transform(
-        COMMAND convert
-        OUTPUT ${loki_idem_stack_srcs}
-        BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
-        DEPENDS ${phys_srcs}
-        MODE idem-stack
-        CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
-        CPP
-        FRONTEND ${LOKI_FRONTEND}
-        SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
-        INCLUDES ${ecwam_intfb_includes}
+      COMMAND convert
+      OUTPUT ${loki_idem_stack_srcs}
+      BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem-stack
+      DEPENDS ${phys_srcs}
+      MODE idem-stack
+      CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki.config
+      CPP
+      FRONTEND ${LOKI_FRONTEND}
+      SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
+      INCLUDES ${ecwam_intfb_includes}
   )
 
   ecbuild_add_library(
@@ -573,9 +553,9 @@ if( HAVE_WAM_LOKI )
       MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_idem_stack
       INSTALL_DIRECTORY module/${ecwam}_idem_stack
   )
-  
+
   ecwam_target_compile_definitions_FILENAME( ${ecwam}_idem_stack )
-  
+
   if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
       target_compile_options( ${ecwam}_idem_stack PUBLIC "-Mcray=pointer" )
   elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
@@ -586,23 +566,13 @@ if( HAVE_WAM_LOKI )
 ############################################################
 ##  Loki SCC transformation:                              ##
 ############################################################
-     foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
-        string(REPLACE ".F90" "" fnc ${src})
-        string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
-        list(APPEND loki_scc_srcs ${fnm})
-     endforeach()
-
-     # loki_transform_convert(
-     #     MODE scc FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
-     #     CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
-     #     PATH ${CMAKE_CURRENT_SOURCE_DIR}
-     #     INCLUDES ${ecwam_intfb_includes}
-     #     OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
-     #     OUTPUT ${loki_scc_srcs}
-     #     DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
-     #  )
-
-     loki_transform(
+    foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
+      string(REPLACE ".F90" "" fnc ${src})
+      string(CONCAT fnm "loki-scc/" ${fnc} ".scc.F90")
+      list(APPEND loki_scc_srcs ${fnm})
+    endforeach()
+
+    loki_transform(
         COMMAND convert
         OUTPUT ${loki_scc_srcs}
         BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc
@@ -613,102 +583,92 @@ if( HAVE_WAM_LOKI )
         FRONTEND ${LOKI_FRONTEND}
         SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
         INCLUDES ${ecwam_intfb_includes}
-     )
-
-     ecbuild_add_library(
-     	TARGET           ${ecwam}_scc
-     	TYPE             ${LIBRARY_TYPE}
-     	DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
-     	SOURCES          ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
-     	PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
-     					 ${MPI_Fortran_LIBRARIES}
-     					 ${${PNAME}_OCEANMODEL_LIBRARIES}
+    )
+
+    ecbuild_add_library(
+       TARGET           ${ecwam}_scc
+       TYPE             ${LIBRARY_TYPE}
+       DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
+       SOURCES          ${ecwam_srcs} ${loki_scc_srcs} ${phys_srcs}
+       PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
+                ${MPI_Fortran_LIBRARIES}
+                ${${PNAME}_OCEANMODEL_LIBRARIES}
                          $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
-     	PRIVATE_LIBS     eccodes_f90
-     					 ${MULTIO_LIBRARIES}
-     					 ${OpenMP_Fortran_LIBRARIES}
-     					 $<${HAVE_FIELD_API}:field_api_${prec}>
-     	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
-     	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
-     )
-     
-     ecwam_target_fortran_module_directory(
-     	TARGET ${ecwam}_scc
-     	MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
-     	INSTALL_DIRECTORY module/${ecwam}_scc
-     )
-     
-     ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc )
-     
-     if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND HAVE_ACC )
-     	target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
-     endif()
+       PRIVATE_LIBS     eccodes_f90
+                ${MULTIO_LIBRARIES}
+                ${OpenMP_Fortran_LIBRARIES}
+                $<${HAVE_FIELD_API}:field_api_${prec}>
+       PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
+       PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
+    )
+
+    ecwam_target_fortran_module_directory(
+       TARGET ${ecwam}_scc
+       MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc
+       INSTALL_DIRECTORY module/${ecwam}_scc
+    )
+
+    ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc )
+
+    if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND HAVE_ACC )
+       target_compile_options( ${ecwam}_scc PUBLIC "-gpu=gvmode,maxregcount:128" )
+    endif()
 
 ############################################################
 ##  Loki SCC with pool allocator for temporaries:         ##
 ############################################################
-#foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
-#        string(REPLACE ".F90" "" fnc ${src})
-#        string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
-#        list(APPEND loki_scc_stack_srcs ${fnm})
-#     endforeach()
-
-     # loki_transform_convert(
-     #     MODE scc-stack FRONTEND ${LOKI_FRONTEND} CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
-     #     CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
-     #     PATH ${CMAKE_CURRENT_SOURCE_DIR}
-     #     INCLUDES ${ecwam_intfb_includes}
-     #     OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
-     #     OUTPUT ${loki_scc_stack_srcs}
-     #     DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
-     # )
-
-     #loki_transform(
-     #   COMMAND convert
-     #   OUTPUT ${loki_scc_stack_srcs}
-     #   BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
-     #   DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
-     #   MODE scc-stack
-     #   CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
-     #   CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
-     #   FRONTEND ${LOKI_FRONTEND}
-     #   SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
-     #   INCLUDES ${ecwam_intfb_includes}
-     #)
-
-     #ecbuild_add_library(
-     # 	TARGET           ${ecwam}_scc_stack
-     #	TYPE             ${LIBRARY_TYPE}
-     # 	DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
-     # 	SOURCES          ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
-     # 	PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
-     #					 ${MPI_Fortran_LIBRARIES}
-     # 					 ${${PNAME}_OCEANMODEL_LIBRARIES}
-     #                    $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
-     # 	PRIVATE_LIBS     eccodes_f90
-     #					 ${MULTIO_LIBRARIES}
-     #					 ${OpenMP_Fortran_LIBRARIES}
-     #					 $<${HAVE_FIELD_API}:field_api_${prec}>
-     #	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
-     #	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
-     #)
-     
-     #ecwam_target_fortran_module_directory(
-     #	TARGET ${ecwam}_scc_stack
-     #	MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
-     #	INSTALL_DIRECTORY module/${ecwam}_scc_stack
-     #)
-     
-     #ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )
-     
-     #if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
-     #   target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
-     #   if( HAVE_ACC )
-     #      target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
-     #   endif()
-     #elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
-     #   target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
-     #endif()
+    foreach(src ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods})
+      string(REPLACE ".F90" "" fnc ${src})
+      string(CONCAT fnm "loki-scc-stack/" ${fnc} ".scc_stack.F90")
+      list(APPEND loki_scc_stack_srcs ${fnm})
+    endforeach()
+
+    loki_transform(
+        COMMAND convert
+        OUTPUT ${loki_scc_stack_srcs}
+        BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-stack
+        DEPENDS ${phys_srcs} wamintgr_loki_gpu.F90 ${global_var_mods}
+        MODE scc-stack
+        CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/ecwam_loki_gpu.config
+        CPP GLOBAL_VAR_OFFLOAD TRIM_VECTOR_SECTIONS
+        FRONTEND ${LOKI_FRONTEND}
+        SOURCES ${CMAKE_CURRENT_SOURCE_DIR}
+        INCLUDES ${ecwam_intfb_includes}
+    )
+
+    ecbuild_add_library(
+        TARGET           ${ecwam}_scc_stack
+        TYPE             ${LIBRARY_TYPE}
+        DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_PHYS_GPU
+        SOURCES          ${ecwam_srcs} ${loki_scc_stack_srcs} ${phys_srcs}
+        PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
+                ${MPI_Fortran_LIBRARIES}
+                 ${${PNAME}_OCEANMODEL_LIBRARIES}
+                         $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
+        PRIVATE_LIBS     eccodes_f90
+                ${MULTIO_LIBRARIES}
+                ${OpenMP_Fortran_LIBRARIES}
+                $<${HAVE_FIELD_API}:field_api_${prec}>
+         PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
+        PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
+    )
+
+    ecwam_target_fortran_module_directory(
+       TARGET ${ecwam}_scc_stack
+       MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_stack
+       INSTALL_DIRECTORY module/${ecwam}_scc_stack
+    )
+
+    ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_stack )
+
+    if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
+      target_compile_options( ${ecwam}_scc_stack PUBLIC "-Mcray=pointer" )
+      if( HAVE_ACC )
+        target_compile_options( ${ecwam}_scc_stack PUBLIC "-gpu=gvmode,maxregcount:128" )
+      endif()
+    elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
+      target_compile_options( ${ecwam}_scc_stack PUBLIC "-fcray-pointer" )
+    endif()
   endif()
 endif()
 
@@ -725,28 +685,28 @@ if( HAVE_CUDA )
   set_source_files_properties( ${wam_scc_cuf_srcs} PROPERTIES COMPILE_OPTIONS "-Mcuda=maxregcount:128" )
 
   ecbuild_add_library(
-  	TARGET           ${ecwam}_scc_cuf
-  	TYPE             ${LIBRARY_TYPE}
-  	DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_CUDA
-  	SOURCES          ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
-  	PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
-  					 ${MPI_Fortran_LIBRARIES}
-  					 ${${PNAME}_OCEANMODEL_LIBRARIES}
+    TARGET           ${ecwam}_scc_cuf
+    TYPE             ${LIBRARY_TYPE}
+    DEFINITIONS      ${ECWAM_DEFINITIONS} WAM_CUDA
+    SOURCES          ${ecwam_srcs} ${wam_scc_cuf_srcs} ${phys_srcs}
+    PUBLIC_LIBS      fiat parkind_${prec} ${ecwam}_intfb
+             ${MPI_Fortran_LIBRARIES}
+             ${${PNAME}_OCEANMODEL_LIBRARIES}
                      $<${HAVE_ACC}:OpenACC::OpenACC_Fortran>
-  	PRIVATE_LIBS     eccodes_f90
-  					 ${MULTIO_LIBRARIES}
-  					 ${OpenMP_Fortran_LIBRARIES}
-  					 $<${HAVE_FIELD_API}:field_api_${prec}>
-  	PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
-  	PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
+    PRIVATE_LIBS     eccodes_f90
+             ${MULTIO_LIBRARIES}
+             ${OpenMP_Fortran_LIBRARIES}
+             $<${HAVE_FIELD_API}:field_api_${prec}>
+    PUBLIC_INCLUDES  $<INSTALL_INTERFACE:include>
+    PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}
   )
 
   ecwam_target_fortran_module_directory(
-  	TARGET ${ecwam}_scc_cuf
-  	MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
-  	INSTALL_DIRECTORY module/${ecwam}_scc_cuf
+    TARGET ${ecwam}_scc_cuf
+    MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${ecwam}_scc_cuf
+    INSTALL_DIRECTORY module/${ecwam}_scc_cuf
   )
-  
+
   ecwam_target_compile_definitions_FILENAME( ${ecwam}_scc_cuf )
 
   target_link_options( ${ecwam}_scc_cuf PUBLIC "-cuda;-gpu=pinned" )
diff --git a/src/programs/CMakeLists.txt b/src/programs/CMakeLists.txt
index 7e921e4a..63865b91 100644
--- a/src/programs/CMakeLists.txt
+++ b/src/programs/CMakeLists.txt
@@ -46,12 +46,12 @@ if( HAVE_WAM_LOKI )
         )
         ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc)
 
-	#  ecbuild_add_executable(
-	#     TARGET          ${PROJECT_NAME}-chief-loki-scc-stack
-	#     SOURCES         chief.F90
-	#     LIBS            ${PROJECT_NAME}_scc_stack ${OpenMP_Fortran_LIBRARIES}
-	# )
-	# ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc-stack)
+        ecbuild_add_executable(
+            TARGET          ${PROJECT_NAME}-chief-loki-scc-stack
+            SOURCES         chief.F90
+            LIBS            ${PROJECT_NAME}_scc_stack ${OpenMP_Fortran_LIBRARIES}
+        )
+        ecwam_target_compile_definitions_FILENAME(${PROJECT_NAME}-chief-loki-scc-stack)
     endif()
 endif()