Porting UFS-WM Noah-MP GSWP3 baseline to land DA workflow

.gitmodules

@@ -6,7 +6,7 @@
 	path = DA_update
         url = https://github.com/ufs-community/land-DA.git
         branch = develop
-[submodule "vector2tile"]
-	path = vector2tile
-	url = https://github.com/NOAA-PSL/land-vector2tile.git
-        branch = develop
+[submodule "tile2tile"]
+	path = tile2tile
+	url = https://github.com/rhaesung/land-tile2tile.git
+        branch = feature/porting

CMakeLists.txt

@@ -44,17 +44,17 @@ include_directories( ${NETCDF_INCLUDE_DIRS} )
 ################################################################################
 set( LANDDA_LINKER_LANGUAGE CXX)
 
-ExternalProject_Add(ufs-weather-model
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model
-  GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model
-  GIT_SUBMODULES_RECURSE TRUE
-  GIT_TAG develop
-  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model
-  CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release
-  INSTALL_COMMAND cmake
-  )
-
-add_subdirectory( vector2tile )
+#ExternalProject_Add(ufs-weather-model
+#  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model
+#  GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model
+#  GIT_SUBMODULES_RECURSE TRUE
+#  GIT_TAG develop
+#  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model
+#  CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release
+#  INSTALL_COMMAND cmake
+#  )
+
+add_subdirectory( tile2tile )
 add_subdirectory( ufs-land-driver-emc-dev )
 add_subdirectory( DA_update/add_jedi_incr )
 

do_submit_cycle.sh

@@ -75,10 +75,10 @@ fi
 ############################
 # set executables
 
-if [[ -e ${BUILDDIR}/bin/vector2tile_converter.exe ]]; then #prefer cmake-built executables
-  export vec2tileexec=${BUILDDIR}/bin/vector2tile_converter.exe
+if [[ -e ${BUILDDIR}/bin/tile2tile_converter.exe ]]; then #prefer cmake-built executables
+  export tile2tileexec=${BUILDDIR}/bin/tile2tile_converter.exe
 else 
-  export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe
+  export tile2tileexec=${CYCLEDIR}/tile2tile/tile2tile_converter.exe
 fi
 if [[ -e ${BUILDDIR}/bin/ufsLand.exe ]]; then
   export LSMexec=${BUILDDIR}/bin/ufsLand.exe
@@ -154,10 +154,12 @@ fi
 ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output 
 
 # copy ICS into restarts, if needed 
-rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
-rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+for tile in 1 2 3 4 5 6
+do
+rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc
+rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc
 # if restart not in experiment out directory, copy the restarts from the ICSDIR
-if [[ ! -e ${rst_out} ]]; then
+ if [[ ! -e ${rst_out} ]]; then
     echo "Looking for ICS: ${rst_in}"
     if [[ -e ${rst_in} ]]; then
        echo "ICS found, copying" 
@@ -166,7 +168,8 @@ if [[ ! -e ${rst_out} ]]; then
        echo "ICS not found. Exiting" 
        exit 10
     fi
-fi
+ fi
+done
 
 # create dates file 
 touch analdates.sh 

submit_cycle.sh

@@ -1,6 +1,6 @@
 #!/bin/bash  
 #SBATCH --job-name=offline_noahmp
-#SBATCH --account=da-cpu
+#SBATCH --account=epic
 #SBATCH --qos=debug
 #SBATCH --nodes=1
 #SBATCH --tasks-per-node=6
@@ -58,7 +58,7 @@ while [ $date_count -lt $cycles_per_job ]; do
     nHH=`echo $NEXTDATE | cut -c9-10`
 
     ############################
-    # copy restarts to workdir, convert to vector for DA (all members) 
+    # copy restarts to workdir, convert to UFS tile for DA (all members) 
 
     mem_ens="mem000" 
 
@@ -67,46 +67,47 @@ while [ $date_count -lt $cycles_per_job ]; do
 
     cd $MEM_WORKDIR
     # copy restarts into work directory
-    rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc
-    if [[ ! -e ${rst_in} ]]; then
-      rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc 
+
+    for tile in 1 2 3 4 5 6
+    do
+    rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc
+    if [[ ! -e ${rst_in} ]]; then  
+      rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc 
     fi
-    rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc
+    rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc
     cp ${rst_in} ${rst_out}
+    done
 
     if [[ $do_jedi == "YES" ]]; then  
-        echo '************************************************'
-        echo 'calling tile2vector' 
-
         export MEM_WORKDIR
 
-        # update vec2tile and tile2vec namelists
-        cp  ${CYCLEDIR}/template.vector2tile vector2tile.namelist
-
-        sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist
-        sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist
-        sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist
-        sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist
-        sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist
-        sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist
-        sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist
-        sed -i -e "s/XXRES/${RES}/g" vector2tile.namelist
-        sed -i -e "s/XXTSTUB/${TSTUB}/g" vector2tile.namelist
-        sed -i -e "s#XXTPATH#${TPATH}#g" vector2tile.namelist
-
-        # submit vec2tile 
+       # update tile2tile namelist
+        cp  ${CYCLEDIR}/template.tile2tile tile2tile.namelist
+
+        sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2tile.namelist
+        sed -i -e "s/XXYYYY/${YYYY}/g" tile2tile.namelist
+        sed -i -e "s/XXMM/${MM}/g" tile2tile.namelist
+        sed -i -e "s/XXDD/${DD}/g" tile2tile.namelist
+        sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist
+        sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist
+        sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" tile2tile.namelist
+        sed -i -e "s/XXRES/${RES}/g" tile2tile.namelist
+        sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2tile.namelist
+        sed -i -e "s#XXTPATH#${TPATH}#g" tile2tile.namelist
+
+       # submit tile2tile 
         echo '************************************************'
-        echo 'calling vector2tile' 
-        
+        echo 'calling tile2tile' 
+
         if [[ $BASELINE =~ 'hera.internal' ]]; then
            source ${CYCLEDIR}/land_mods
         fi 
-        $vec2tileexec vector2tile.namelist
+        $tile2tileexec tile2tile.namelist
         if [[ $? != 0 ]]; then
-            echo "vec2tile failed"
+            echo "tile2tile failed"
             exit 
         fi
-    fi # vector2tile for DA
+    fi # tile2tile for DA
 
     ############################
     # do DA update
@@ -126,9 +127,10 @@ while [ $date_count -lt $cycles_per_job ]; do
             exit
         fi   
     fi 
-
+
+    exit    
     ############################
-    #  convert back to vector, run model (all members) convert back to vector, run model (all members)
+    #  convert back to UFS tile, run model (all members)
 
     mem_ens="mem000" 
 
@@ -139,7 +141,7 @@ while [ $date_count -lt $cycles_per_job ]; do
 
     if [[ $do_jedi == "YES" ]]; then  
         echo '************************************************'
-        echo 'calling tile2vector' 
+        echo 'calling tile2tile' 
 
         if [[ $BASELINE =~ 'hera.internal' ]]; then
            source ${CYCLEDIR}/land_mods

template.vector2tile → template.tile2tile

@@ -4,7 +4,7 @@
 ! Direction of conversion: either "vector2tile" or "tile2vector" for restart file
 !                                 "lndp2tile" or "lndp2vector" for perturbation 
 
-  direction = "vector2tile"
+  direction = "ufs2jedi"
 
 ! FV3 resolution and path to oro files for restart/perturbation conversion
 
@@ -26,8 +26,8 @@
 
 ! Location of tile restart files (tile2vector direction)
 
-  tile_restart_path = "junk"
-
+!  tile_restart_path = "junk"
+  tile_restart_path = "./"  
 ! Path for converted files; if same as tile/vector path, files may be overwritten
 
   output_path = "./"

tile2tile/.gitignore

@@ -0,0 +1,4 @@
+vector2tile_converter.exe
+vector2tile_converter.o
+*.mod 
+*.o

tile2tile/CMakeLists.txt

@@ -0,0 +1,75 @@
+# (C) Copyright 2022 .
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+cmake_minimum_required( VERSION 3.12)
+project( tile2tile VERSION 2022.10 LANGUAGES Fortran )
+
+# ecbuild integration
+find_package(ecbuild 3.3.2 REQUIRED)
+include( ecbuild_system NO_POLICY_SCOPE )
+ecbuild_declare_project()
+
+list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+#include( tile2tile_compiler_flags )
+
+################################################################################
+# Dependencies
+################################################################################
+
+find_package(OpenMP COMPONENTS C Fortran)
+find_package(MPI REQUIRED COMPONENTS C Fortran)
+find_package(NetCDF REQUIRED COMPONENTS Fortran )
+
+################################################################################
+# Sources
+################################################################################
+
+set( TILE2TILE_LINKER_LANGUAGE Fortran )
+
+list ( APPEND src_files
+./namelist_mod.f90
+./tile2tile_restart_mod.f90
+)
+
+ecbuild_add_library( TARGET   tile2tile
+                     SOURCES  ${src_files}
+                     INSTALL_HEADERS LISTED
+                     LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE}
+                    )
+
+
+target_link_libraries(tile2tile PUBLIC NetCDF::NetCDF_Fortran)
+target_link_libraries(tile2tile PUBLIC MPI::MPI_Fortran)
+target_link_libraries(tile2tile PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_Fortran)
+
+# Fortran module output directory for build and install interfaces
+set(MODULE_DIR module/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION})
+set_target_properties(${PROJECT_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR})
+install(DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}/ DESTINATION ${MODULE_DIR})
+target_include_directories(${PROJECT_NAME} INTERFACE
+                            $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/${MODULE_DIR}>
+                            $<INSTALL_INTERFACE:${MODULE_DIR}>)
+
+set( TILE2TILE_INCLUDE_DIRS  ${CMAKE_Fortran_MODULE_DIRECTORY} )
+set( TILE2TILE_LIBRARIES tile2tile )
+
+# tile2tile_converter.exe executable
+#-------------------------------------------------------------------------------
+set ( exe_files tile2tile_driver.f90 )
+
+ecbuild_add_executable( TARGET   tile2tile_converter.exe
+                        SOURCES  ${exe_files}
+                        LIBS     tile2tile
+                        LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE}
+                       )
+
+
+################################################################################
+# Finalise configuration
+################################################################################
+
+ecbuild_install_project( NAME tile2tile )
+ecbuild_print_summary()

tile2tile/Makefile

@@ -0,0 +1,24 @@
+# Makefile 
+#
+.SUFFIXES:
+.SUFFIXES: .o .f90
+
+include ./user_build_config
+
+OBJS =	namelist_mod.o vector2tile_restart_mod.o vector2tile_perturbation_mod.o vector2tile_driver.o
+
+all:	vector2tile_converter.exe
+
+.f90.o:
+	$(COMPILERF90) -c $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(*).f90
+
+vector2tile_converter.exe: $(OBJS)
+	$(COMPILERF90) -o $(@) $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(OBJS) $(NETCDFLIB)
+
+clean:
+	rm -f *.o *.mod *.exe
+
+
+#
+# Dependencies:
+#

tile2tile/README.md

@@ -0,0 +1,30 @@
+Code to map between vector format used by the Noah-MP offline driver, and the tile format used by the UFS atmospheric model. Currently used to prepare input tile files for JEDI. These files include only those fields required by JEDI, rather than the full restart. Can also be used to map stochastic_physics output to the tile or vector. Stochastic physics output files depend on the layout, with 1x1 layout giving one file per tile.
+
+Mike Barlage, Clara Draper. Dec 2021.
+
+To compile on hera: 
+
+>configure
+
+ choose hera
+
+ load the modules indicated
+
+>make 
+
+To run: 
+
+>vector2tile_converter.exe namelist.vector2tile
+
+the namelist defines the conversion direction and the paths of the files
+
+Details: 
+
+the vector2tile pathway assumes that the vector file exists in the vector_restart_path directory and overwrites/creates tile files in the output_path
+
+the tile2vector pathway is a little tricky, it assumes the tile files exist in tile_restart_path and overwrites only the snow variables in the vector file in the output_path. If the vector file does not exist in output_path the process will fail.  
+
+the overall assumption here is that we will have a full model vector restart file, then convert the vector to tiles for only snow variables, then convert the updated snow variables back to the full vector restart file
+
+For the lndp2vector or lndp2tile option, a new file will be created with the pertbations and lat lon only.
+