Enable building of coupled model (ufs-community#217)

* Option to build coupled model
* fix in build.sh
* update modulefiles for orion, hera and wcoss to load pio.  Make PIO optional in CMakeLists.txt if not building S2S
* FMS needs to be linked in MOM6 objects. Add 32BIT and S2S check
* add S2S option to MAKE_OPT
* Add standard s2s tests
* add CICE_IO option (default: NetCDF). Add MOM6 solo executable option (default: OFF) MOM6 library and NUOPC cap are 2 separate entities.
* add debug flags for MOM6 and CICE from @MinsukJi-NOAA
* add debug modulefiles for hera, orion and wcoss_dell_p3
* enable VERBOSE on build only when requested.  Remove -warn flags from CMEPS and MOM6 as they drown out any error messages.  The warnings should be resolved at the upstream repos.  When building with -DDEBUG=ON, set CMAKE_BUILD_TYPE=Debug.

Co-authored-by: Dusan Jovic <[email protected]>

.gitignore

@@ -1,18 +1,70 @@
-coupled*.mk
-rt.conf.single
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+*.x
+
+# system files
 *.swp
 *~
+.DS_Store
 
+# python files
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# CMake files and directories
 build*/
 install*/
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
 
-ufs_weather_model
-NEMS.exe
-*.exe
-
-tests/fv3_*.exe
+# repo-specific ignorables
+coupled*.mk
+rt.conf.single
+ufs_model
 tests/modules.fv3_*
 tests/ecflow_run/
 tests/log_*/
 tests/lock/
 tests/Compile_*.log
+tests/fail_test
+log*
+build*.log*

.gitmodules

@@ -18,3 +18,19 @@
   path = stochastic_physics
   url = https://github.com/noaa-psd/stochastic_physics
   branch = master
+[submodule "CMakeModules"]
+  path = CMakeModules
+  url = https://github.com/NOAA-EMC/CMakeModules
+  branch = develop
+[submodule "CMEPS"]
+  path = CMEPS-interface/CMEPS
+  url = https://github.com/NOAA-EMC/CMEPS.git
+  branch = emc/develop
+[submodule "MOM6"]
+  path = MOM6-interface/MOM6
+  url = https://github.com/NOAA-EMC/MOM6
+  branch = dev/emc
+[submodule "CICE"]
+  path = CICE-interface/CICE
+  url = https://github.com/NOAA-EMC/CICE
+  branch = emc/develop

CICE-interface/CMakeLists.txt

@@ -0,0 +1,101 @@
+### CICE Fortran compiler flags
+if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+  set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -ffree-line-length-none ")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
+  set(CMAKE_Fortran_LINK_FLAGS "" )
+elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
+  set(CMAKE_Fortran_FLAGS "-g -traceback")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -FR -convert big_endian -assume byterecl -ftz -align array64byte -xHOST")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg")
+  set(CMAKE_Fortran_LINK_FLAGS "")
+else()
+  message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
+endif()
+
+### CICE C compiler flags
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+  set(CMAKE_C_FLAGS "-g")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+  set(CMAKE_C_FLAGS_RELEASE "-O3")
+  set(CMAKE_C_FLAGS_DEBUG "-O0")
+  set(CMAKE_C_LINK_FLAGS "")
+elseif(CMAKE_C_COMPILER_ID MATCHES "Intel")
+  set( CMAKE_C_FLAGS "-g -traceback")
+  set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xHOST")
+  set( CMAKE_C_FLAGS_RELEASE "-O2 -fp-model precise")
+  set( CMAKE_C_FLAGS_DEBUG "-O0 -ftrapuv")
+  set( CMAKE_C_LINK_FLAGS "")
+elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
+  set(CMAKE_C_FLAGS "-g")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+  set( CMAKE_C_FLAGS_RELEASE "-O3" )
+  set( CMAKE_C_FLAGS_DEBUG "-O0" )
+  set( CMAKE_C_LINK_FLAGS "" )
+else()
+  message(WARNING "C compiler with ID ${CMAKE_C_COMPILER_ID} will be used with CMake default options")
+endif()
+
+# Configuration Options
+set(CICE_IO "NetCDF" CACHE STRING "CICE OPTIONS: Choose IO options.")
+set_property(CACHE CICE_IO PROPERTY STRINGS "NetCDF" "PIO" "Binary")
+
+# Too many files to list, so include them via this file
+include("cice_files.cmake")
+
+# Collect source files for library
+list(APPEND lib_src_files
+  ${cice_shared_files}
+  ${cice_shared_files_c}
+  ${icepack_files}
+  ${cice_mpi_comm_files}
+  ${cice_nuopc_cmeps_driver_files})
+
+list(APPEND _cice_defs FORTRANUNDERSCORE
+                       coupled)
+
+# Select IO source files based on CICE_IO
+if(CICE_IO MATCHES "NetCDF")
+  list(APPEND lib_src_files ${cice_netcdf_io_files})
+  list(APPEND _cice_defs USE_NETCDF)
+elseif(CICE_IO MATCHES "PIO")
+  list(APPEND lib_src_files ${cice_pio2_io_files})
+  list(APPEND _cice_defs USE_NETCDF)
+elseif(CICE_IO MATCHES "Binary")
+  list(APPEND lib_src_files ${cice_binary_io_files})
+endif()
+
+### Create target library and set PUBLIC interfaces on the library
+add_library(cice STATIC ${lib_src_files})
+set_target_properties(cice PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_compile_definitions(cice PUBLIC "${_cice_defs}")
+target_include_directories(cice PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
+                                       $<INSTALL_INTERFACE:mod>)
+target_link_libraries(cice PUBLIC esmf)
+if(CICE_IO MATCHES "^(NetCDF|PIO)$")
+  target_link_libraries(cice PUBLIC NetCDF::NetCDF_Fortran)
+  if(CICE_IO MATCHES "PIO")
+    target_link_libraries(cice PUBLIC PIO::PIO_Fortran)
+  endif()
+endif()
+if(OpenMP_Fortran_FOUND)
+  target_link_libraries(cice PRIVATE OpenMP::OpenMP_Fortran)
+endif()
+
+###############################################################################
+### Install
+###############################################################################
+
+install(
+  TARGETS cice
+  EXPORT  cice-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  COMPONENT Library)
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX}/CICE)
+
+install(EXPORT      cice-config
+        DESTINATION lib/cmake)

CICE-interface/cice_files.cmake

@@ -0,0 +1,185 @@
+list(APPEND cice_shared_files
+  #Shared List:
+  CICE/cicecore/shared/ice_arrays_column.F90
+  CICE/cicecore/shared/ice_calendar.F90
+  CICE/cicecore/shared/ice_constants.F90
+  CICE/cicecore/shared/ice_distribution.F90
+  CICE/cicecore/shared/ice_domain_size.F90
+  CICE/cicecore/shared/ice_fileunits.F90
+  CICE/cicecore/shared/ice_init_column.F90
+  CICE/cicecore/shared/ice_kinds_mod.F90
+  CICE/cicecore/shared/ice_restart_column.F90
+  CICE/cicecore/shared/ice_restart_shared.F90
+  CICE/cicecore/shared/ice_spacecurve.F90
+
+  #Analysis List:
+  CICE/cicecore/cicedynB/analysis/ice_diagnostics.F90
+  CICE/cicecore/cicedynB/analysis/ice_diagnostics_bgc.F90
+  CICE/cicecore/cicedynB/analysis/ice_history.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_bgc.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_drag.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_fsd.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_mechred.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_pond.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_shared.F90
+
+  #Dynamics List:
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_eap.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_evp.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_shared.F90
+  CICE/cicecore/cicedynB/dynamics/ice_transport_driver.F90
+  CICE/cicecore/cicedynB/dynamics/ice_transport_remap.F90
+
+  #General List:
+  CICE/cicecore/cicedynB/general/ice_flux.F90
+  CICE/cicecore/cicedynB/general/ice_flux_bgc.F90
+  CICE/cicecore/cicedynB/general/ice_forcing.F90
+  CICE/cicecore/cicedynB/general/ice_forcing_bgc.F90
+  CICE/cicecore/cicedynB/general/ice_init.F90
+  CICE/cicecore/cicedynB/general/ice_state.F90
+  CICE/cicecore/cicedynB/general/ice_step_mod.F90
+
+  #Infrastructure List
+  CICE/cicecore/cicedynB/infrastructure/ice_blocks.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_domain.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_grid.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_read_write.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_restart_driver.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_restoring.F90
+)
+
+
+#Icepack List:
+list(APPEND icepack_files
+  CICE/icepack/columnphysics/icepack_aerosol.F90
+  CICE/icepack/columnphysics/icepack_age.F90
+  CICE/icepack/columnphysics/icepack_algae.F90
+  CICE/icepack/columnphysics/icepack_atmo.F90
+  CICE/icepack/columnphysics/icepack_brine.F90
+  CICE/icepack/columnphysics/icepack_firstyear.F90
+  CICE/icepack/columnphysics/icepack_flux.F90
+  CICE/icepack/columnphysics/icepack_fsd.F90
+  CICE/icepack/columnphysics/icepack_intfc.F90
+  CICE/icepack/columnphysics/icepack_isotope.F90
+  CICE/icepack/columnphysics/icepack_itd.F90
+  CICE/icepack/columnphysics/icepack_kinds.F90
+  CICE/icepack/columnphysics/icepack_mechred.F90
+  CICE/icepack/columnphysics/icepack_meltpond_cesm.F90
+  CICE/icepack/columnphysics/icepack_meltpond_lvl.F90
+  CICE/icepack/columnphysics/icepack_meltpond_topo.F90
+  CICE/icepack/columnphysics/icepack_mushy_physics.F90
+  CICE/icepack/columnphysics/icepack_ocean.F90
+  CICE/icepack/columnphysics/icepack_orbital.F90
+  CICE/icepack/columnphysics/icepack_parameters.F90
+  CICE/icepack/columnphysics/icepack_shortwave.F90
+  CICE/icepack/columnphysics/icepack_therm_0layer.F90
+  CICE/icepack/columnphysics/icepack_therm_bl99.F90
+  CICE/icepack/columnphysics/icepack_therm_itd.F90
+  CICE/icepack/columnphysics/icepack_therm_mushy.F90
+  CICE/icepack/columnphysics/icepack_therm_shared.F90
+  CICE/icepack/columnphysics/icepack_therm_vertical.F90
+  CICE/icepack/columnphysics/icepack_tracers.F90
+  CICE/icepack/columnphysics/icepack_warnings.F90
+  CICE/icepack/columnphysics/icepack_wavefracspec.F90
+  CICE/icepack/columnphysics/icepack_zbgc.F90
+  CICE/icepack/columnphysics/icepack_zbgc_shared.F90
+  CICE/icepack/columnphysics/icepack_zsalinity.F90
+)
+
+list(APPEND cice_shared_files_c
+  CICE/cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c
+)
+
+#-- Using MPI
+list(APPEND cice_mpi_comm_files
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_boundary.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_broadcast.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_communicate.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_exit.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_global_reductions.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90
+)
+
+#-- Using Serial
+list(APPEND cice_serial_comm_files
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_broadcast.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_communicate.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_exit.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_gather_scatter.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_global_reductions.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90
+)
+
+#-- Using binary IO
+list(APPEND cice_binary_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90
+)
+
+#-- Using NetCDF IO
+list(APPEND cice_netcdf_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90
+)
+
+#PIO2 I/O List:
+list(APPEND cice_pio2_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_pio.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90
+)
+
+#-- Using standalone driver
+list(APPEND cice_standalone_driver_files
+  CICE/cicecore/drivers/standalone/cice/CICE.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_FinalMod.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_InitMod.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_RunMod.F90
+)
+
+#-- Using NUOPC CMEPS driver
+list(APPEND cice_nuopc_cmeps_driver_files
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_FinalMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_InitMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_RunMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/cice_wrapper_mod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_comp_nuopc.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_import_export.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_scam.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_shr_methods.F90
+)
+
+#-- Using NUOPC DMI driver
+list(APPEND cice_nuopc_dmi_driver_files
+  CICE/cicecore/drivers/nuopc/dmi/CICE.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_FinalMod.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_InitMod.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90
+)
+
+#-- Using direct driver
+list(APPEND cice_direct_driver_files
+  CICE/cicecore/drivers/direct/hadgem3/CICE.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_FinalMod.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_InitMod.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_RunMod.F90
+)
+
+#-- Using MCT driver
+list(APPEND cice_mct_driver_files
+  CICE/cicecore/drivers/mct/cesm1/CICE_FinalMod.F90
+  CICE/cicecore/drivers/mct/cesm1/CICE_InitMod.F90
+  CICE/cicecore/drivers/mct/cesm1/CICE_RunMod.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_comp_esmf.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_comp_mct.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_cpl_indices.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_import_export.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_prescribed_mod.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_scam.F90
+)