From 5a22db7d9dd70e52d9934416e3ecb012d4cce768 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Fri, 15 Sep 2023 21:04:54 +0000 Subject: [PATCH 01/65] update gitmodules --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 71f29012..50b32fde 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,5 +8,5 @@ branch = develop [submodule "vector2tile"] path = vector2tile - url = https://github.com/NOAA-PSL/land-vector2tile.git - branch = develop + url = https://github.com/rhaesung/land-vector2tile.git + branch = porting From cbab24fd3c37529630a8b0b503236f617cca9920 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Fri, 15 Sep 2023 21:22:01 +0000 Subject: [PATCH 02/65] update vector2tile --- vector2tile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector2tile b/vector2tile index 2fb5c1b4..8a426523 160000 --- a/vector2tile +++ b/vector2tile @@ -1 +1 @@ -Subproject commit 2fb5c1b493b3c03809e081f6fe8e8e76a40a100d +Subproject commit 8a4265234304aa7bded61e3358a181f0241bda66 From 3a1bd757c6906bc721820b79595b06b5f3365263 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:05:49 +0000 Subject: [PATCH 03/65] add template.tile2tile --- template.tile2tile | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 template.tile2tile diff --git a/template.tile2tile b/template.tile2tile new file mode 100644 index 00000000..084195be --- /dev/null +++ b/template.tile2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "ufs2jedi" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + +! tile_restart_path = "junk" + tile_restart_path = "./" +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ From 65d42a1dd0c505c21186cfd4876ec82bd60105d9 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:06:40 +0000 Subject: [PATCH 04/65] remove vector2tile submodule --- vector2tile | 1 - 1 file changed, 1 deletion(-) delete mode 160000 vector2tile diff --git a/vector2tile b/vector2tile deleted file mode 160000 index 8a426523..00000000 --- a/vector2tile +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8a4265234304aa7bded61e3358a181f0241bda66 From 97309ba22785d4bb43b3ee59112550cd230e709d Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:07:08 +0000 Subject: [PATCH 05/65] delete template for vector2tile --- template.vector2tile | 52 -------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 template.vector2tile diff --git a/template.vector2tile b/template.vector2tile deleted file mode 100644 index e175d2e0..00000000 --- a/template.vector2tile +++ /dev/null @@ -1,52 +0,0 @@ -&run_setup - -!------------------- common ------------------- -! Direction of conversion: either "vector2tile" or "tile2vector" for restart file -! "lndp2tile" or "lndp2vector" for perturbation - - direction = "vector2tile" - -! FV3 resolution and path to oro files for restart/perturbation conversion - - tile_size = XXRES - tile_path = "XXTPATH" - tile_fstub = "XXTSTUB" - -!------------------- only restart conversion ------------------- -! Time stamp for conversion for restart conversion - - restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" - -! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" - -! Location of vector restart file (vector2tile direction) - - vector_restart_path = "./" - -! Location of tile restart files (tile2vector direction) - - tile_restart_path = "junk" - -! Path for converted files; if same as tile/vector path, files may be overwritten - - output_path = "./" - -!------------------- only perturbation mapping ------------------- -! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file - - lndp_layout = "" - -! input perturbation pattern files - - lndp_input_file = "" - -! output files - - lndp_output_file = "" - -! land perturbation variable list, for an example: 'vgf','smc' - - lndp_var_list = '' - -/ From 693e514b14008578452e4ca3782cec6399be20c0 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:07:27 +0000 Subject: [PATCH 06/65] update .gitmodules --- .gitmodules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 50b32fde..949965c6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ path = DA_update url = https://github.com/ufs-community/land-DA.git branch = develop -[submodule "vector2tile"] - path = vector2tile - url = https://github.com/rhaesung/land-vector2tile.git - branch = porting +[submodule "tile2tile"] + path = tile2tile + url = https://github.com/rhaesung/land-tile2tile.git + branch = feature/porting From 602eba8770bd58e24c780ac1325803ea4033d795 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:07:58 +0000 Subject: [PATCH 07/65] update CMakeLists.txt --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03cae5a8..c8641ea8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,17 +44,17 @@ include_directories( ${NETCDF_INCLUDE_DIRS} ) ################################################################################ set( LANDDA_LINKER_LANGUAGE CXX) -ExternalProject_Add(ufs-weather-model - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model - GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model - GIT_SUBMODULES_RECURSE TRUE - GIT_TAG develop - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model - CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release - INSTALL_COMMAND cmake - ) - -add_subdirectory( vector2tile ) +#ExternalProject_Add(ufs-weather-model +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model +# GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model +# GIT_SUBMODULES_RECURSE TRUE +# GIT_TAG develop +# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model +# CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release +# INSTALL_COMMAND cmake +# ) + +add_subdirectory( tile2tile ) add_subdirectory( ufs-land-driver-emc-dev ) add_subdirectory( DA_update/add_jedi_incr ) From 92f89d94045b6a4ca261051cd95de38b396ad1f1 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:08:29 +0000 Subject: [PATCH 08/65] add tile2tile submodule --- tile2tile/.gitignore | 4 + tile2tile/CMakeLists.txt | 75 +++ tile2tile/Makefile | 24 + tile2tile/README.md | 30 + tile2tile/cmake/compiler_flags_GNU_CXX.cmake | 45 ++ .../cmake/compiler_flags_GNU_Fortran.cmake | 43 ++ .../compiler_flags_GNU_Fortran.cmake_test | 45 ++ .../cmake/compiler_flags_Intel_Fortran.cmake | 41 ++ .../cmake/vector2tile_compiler_flags.cmake | 23 + .../config/user_build_config.gfortran.serial | 11 + tile2tile/config/user_build_config.hera | 10 + tile2tile/configure | 39 ++ tile2tile/namelist.tile2tile | 52 ++ tile2tile/namelist_mod.f90 | 86 +++ tile2tile/pull_request_template.md | 32 + tile2tile/tile2tile_driver.f90 | 48 ++ tile2tile/tile2tile_restart_mod.f90 | 637 ++++++++++++++++++ tile2tile/user_build_config | 10 + 18 files changed, 1255 insertions(+) create mode 100644 tile2tile/.gitignore create mode 100644 tile2tile/CMakeLists.txt create mode 100644 tile2tile/Makefile create mode 100644 tile2tile/README.md create mode 100755 tile2tile/cmake/compiler_flags_GNU_CXX.cmake create mode 100755 tile2tile/cmake/compiler_flags_GNU_Fortran.cmake create mode 100755 tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test create mode 100755 tile2tile/cmake/compiler_flags_Intel_Fortran.cmake create mode 100755 tile2tile/cmake/vector2tile_compiler_flags.cmake create mode 100644 tile2tile/config/user_build_config.gfortran.serial create mode 100644 tile2tile/config/user_build_config.hera create mode 100755 tile2tile/configure create mode 100644 tile2tile/namelist.tile2tile create mode 100644 tile2tile/namelist_mod.f90 create mode 100644 tile2tile/pull_request_template.md create mode 100644 tile2tile/tile2tile_driver.f90 create mode 100644 tile2tile/tile2tile_restart_mod.f90 create mode 100644 tile2tile/user_build_config diff --git a/tile2tile/.gitignore b/tile2tile/.gitignore new file mode 100644 index 00000000..18de9433 --- /dev/null +++ b/tile2tile/.gitignore @@ -0,0 +1,4 @@ +vector2tile_converter.exe +vector2tile_converter.o +*.mod +*.o diff --git a/tile2tile/CMakeLists.txt b/tile2tile/CMakeLists.txt new file mode 100644 index 00000000..5e52d289 --- /dev/null +++ b/tile2tile/CMakeLists.txt @@ -0,0 +1,75 @@ +# (C) Copyright 2022 . +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +cmake_minimum_required( VERSION 3.12) +project( tile2tile VERSION 2022.10 LANGUAGES Fortran ) + +# ecbuild integration +find_package(ecbuild 3.3.2 REQUIRED) +include( ecbuild_system NO_POLICY_SCOPE ) +ecbuild_declare_project() + +list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +#include( tile2tile_compiler_flags ) + +################################################################################ +# Dependencies +################################################################################ + +find_package(OpenMP COMPONENTS C Fortran) +find_package(MPI REQUIRED COMPONENTS C Fortran) +find_package(NetCDF REQUIRED COMPONENTS Fortran ) + +################################################################################ +# Sources +################################################################################ + +set( TILE2TILE_LINKER_LANGUAGE Fortran ) + +list ( APPEND src_files +./namelist_mod.f90 +./tile2tile_restart_mod.f90 +) + +ecbuild_add_library( TARGET tile2tile + SOURCES ${src_files} + INSTALL_HEADERS LISTED + LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE} + ) + + +target_link_libraries(tile2tile PUBLIC NetCDF::NetCDF_Fortran) +target_link_libraries(tile2tile PUBLIC MPI::MPI_Fortran) +target_link_libraries(tile2tile PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_Fortran) + +# Fortran module output directory for build and install interfaces +set(MODULE_DIR module/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}) +set_target_properties(${PROJECT_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}) +install(DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}/ DESTINATION ${MODULE_DIR}) +target_include_directories(${PROJECT_NAME} INTERFACE + $ + $) + +set( TILE2TILE_INCLUDE_DIRS ${CMAKE_Fortran_MODULE_DIRECTORY} ) +set( TILE2TILE_LIBRARIES tile2tile ) + +# tile2tile_converter.exe executable +#------------------------------------------------------------------------------- +set ( exe_files tile2tile_driver.f90 ) + +ecbuild_add_executable( TARGET tile2tile_converter.exe + SOURCES ${exe_files} + LIBS tile2tile + LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE} + ) + + +################################################################################ +# Finalise configuration +################################################################################ + +ecbuild_install_project( NAME tile2tile ) +ecbuild_print_summary() diff --git a/tile2tile/Makefile b/tile2tile/Makefile new file mode 100644 index 00000000..45308c7e --- /dev/null +++ b/tile2tile/Makefile @@ -0,0 +1,24 @@ +# Makefile +# +.SUFFIXES: +.SUFFIXES: .o .f90 + +include ./user_build_config + +OBJS = namelist_mod.o vector2tile_restart_mod.o vector2tile_perturbation_mod.o vector2tile_driver.o + +all: vector2tile_converter.exe + +.f90.o: + $(COMPILERF90) -c $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(*).f90 + +vector2tile_converter.exe: $(OBJS) + $(COMPILERF90) -o $(@) $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(OBJS) $(NETCDFLIB) + +clean: + rm -f *.o *.mod *.exe + + +# +# Dependencies: +# diff --git a/tile2tile/README.md b/tile2tile/README.md new file mode 100644 index 00000000..69c46d60 --- /dev/null +++ b/tile2tile/README.md @@ -0,0 +1,30 @@ +Code to map between vector format used by the Noah-MP offline driver, and the tile format used by the UFS atmospheric model. Currently used to prepare input tile files for JEDI. These files include only those fields required by JEDI, rather than the full restart. Can also be used to map stochastic_physics output to the tile or vector. Stochastic physics output files depend on the layout, with 1x1 layout giving one file per tile. + +Mike Barlage, Clara Draper. Dec 2021. + +To compile on hera: + +>configure + + choose hera + + load the modules indicated + +>make + +To run: + +>vector2tile_converter.exe namelist.vector2tile + +the namelist defines the conversion direction and the paths of the files + +Details: + +the vector2tile pathway assumes that the vector file exists in the vector_restart_path directory and overwrites/creates tile files in the output_path + +the tile2vector pathway is a little tricky, it assumes the tile files exist in tile_restart_path and overwrites only the snow variables in the vector file in the output_path. If the vector file does not exist in output_path the process will fail. + +the overall assumption here is that we will have a full model vector restart file, then convert the vector to tiles for only snow variables, then convert the updated snow variables back to the full vector restart file + +For the lndp2vector or lndp2tile option, a new file will be created with the pertbations and lat lon only. + diff --git a/tile2tile/cmake/compiler_flags_GNU_CXX.cmake b/tile2tile/cmake/compiler_flags_GNU_CXX.cmake new file mode 100755 index 00000000..dbe13c02 --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_CXX.cmake @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +if( HAVE_OMP ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fopenmp") +else( ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fno-openmp") +endif( ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_DEBUG "-O0 -g" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_BIT "-O2" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_CXX_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake new file mode 100755 index 00000000..c0b618cc --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake @@ -0,0 +1,43 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test new file mode 100755 index 00000000..95f6209b --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -fbacktrace -ffree-line-length-none -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -ffree-form") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) +set( CMAKE_Fortran_FLAGS_RELEASE " " ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake b/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake new file mode 100755 index 00000000..7eef4052 --- /dev/null +++ b/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake @@ -0,0 +1,41 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -convert big_endian -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -sox -traceback -msse2" ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3 -debug minimal -fp-model source" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -debug minimal -fp-model source" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/tile2tile/cmake/vector2tile_compiler_flags.cmake b/tile2tile/cmake/vector2tile_compiler_flags.cmake new file mode 100755 index 00000000..8aebfaec --- /dev/null +++ b/tile2tile/cmake/vector2tile_compiler_flags.cmake @@ -0,0 +1,23 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#add_definitions ( -Duse_libMPI -Duse_netCDF -DSPMD -DNXGLOB=4) +add_definitions ( -Duse_libMPI -Duse_netCDF -Dncdf -Dgather_scatter_barrier) + +####################################################################################### +# Fortran +####################################################################################### + +if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) + include( compiler_flags_GNU_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) + include( compiler_flags_Intel_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "XL" ) + include( compiler_flags_XL_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" ) + include( compiler_flags_Cray_Fortran ) +else() + message( STATUS "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options") +endif() diff --git a/tile2tile/config/user_build_config.gfortran.serial b/tile2tile/config/user_build_config.gfortran.serial new file mode 100644 index 00000000..a7191d5c --- /dev/null +++ b/tile2tile/config/user_build_config.gfortran.serial @@ -0,0 +1,11 @@ + +#=============================================================================== +# Placeholder options for Mac gfortran +#=============================================================================== + + COMPILERF90 = /opt/local/bin/gfortran-mp-11 + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/opt/local/include + NETCDFLIB = -L/opt/local/lib -lnetcdf -lnetcdff + diff --git a/tile2tile/config/user_build_config.hera b/tile2tile/config/user_build_config.hera new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/tile2tile/config/user_build_config.hera @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/tile2tile/configure b/tile2tile/configure new file mode 100755 index 00000000..24aad730 --- /dev/null +++ b/tile2tile/configure @@ -0,0 +1,39 @@ +#!/usr/bin/perl + + if($#ARGV == 0) { + $response = shift(@ARGV) ; + print("Configure soil test environment: $response \n"); + }else { + print "Please select from following predefined environments: \n\n"; + + print " 1. hera \n"; + print " 2. orion \n"; + print " 3. gfortran compiler serial \n"; + print " 0. exit only \n"; + + printf "\nEnter selection : " ; + + $response = ; + chop($response); + } + + if ($response == 1) { + # Hera settings + system "cp config/user_build_config.hera user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load 2022.2.0 netcdf/4.7.0 \n\n\n"; + } + + elsif ($response == 2) { + # Orion settings + system "cp config/user_build_config.orion user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load intel netcdf \n\n\n"; + } + + elsif ($response == 3) { + # Mike's Mac settings + system "cp config/user_build_config.gfortran.serial user_build_config"; + } + + else {print "no selection $response\n"; last} diff --git a/tile2tile/namelist.tile2tile b/tile2tile/namelist.tile2tile new file mode 100644 index 00000000..a7a0b680 --- /dev/null +++ b/tile2tile/namelist.tile2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "vector2tile" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/restart/" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x4" + +! input files + + lndp_input_file = "/scratch2/NCEPDEV/land/data/DA/ensemble_pert/workg_T162_984x488.tileXX.nc" + +! output files + + lndp_output_file = "./output.nc" + +! land perturbation variable list + + lndp_var_list='vgf','smc' + +/ diff --git a/tile2tile/namelist_mod.f90 b/tile2tile/namelist_mod.f90 new file mode 100644 index 00000000..4fbca14b --- /dev/null +++ b/tile2tile/namelist_mod.f90 @@ -0,0 +1,86 @@ +module namelist_mod + implicit none + + integer, parameter :: max_n_var_lndp = 20 + type namelist_type + character*256 :: namelist_name = "" + character*11 :: direction = "" + character*256 :: tile_path = "" + character*256 :: tile_fstub = "" + integer :: tile_size + character*19 :: restart_date = "" + character*256 :: vector_restart_path = "" + character*256 :: tile_restart_path = "" + character*256 :: output_path = "" + character*256 :: static_filename = "" + character*3 :: lndp_layout = "" + character*256 :: lndp_input_file = "" + character*256 :: lndp_output_file = "" + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + end type namelist_type + +contains + + subroutine ReadNamelist(namelist) + + type(namelist_type) :: namelist + character*11 :: direction + character*256 :: tile_path + character*256 :: tile_fstub + integer :: tile_size + character*19 :: restart_date + character*256 :: vector_restart_path + character*256 :: tile_restart_path + character*256 :: output_path + character*256 :: static_filename + character*3 :: lndp_layout + character*256 :: lndp_input_file + character*256 :: lndp_output_file + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + integer :: k + + namelist / run_setup / direction, tile_path, tile_fstub, tile_size, restart_date, vector_restart_path, & + tile_restart_path, output_path, static_filename, lndp_layout, & + lndp_input_file, lndp_output_file, lndp_var_list, n_var_lndp + + lndp_var_list = 'XXX' + + open(30, file=namelist%namelist_name, form="formatted") + read(30, run_setup) + close(30) + + namelist%direction = direction + namelist%tile_path = tile_path + namelist%tile_fstub = tile_fstub + namelist%tile_size = tile_size + namelist%restart_date = restart_date + namelist%vector_restart_path = vector_restart_path + namelist%tile_restart_path = tile_restart_path + namelist%output_path = output_path + namelist%static_filename = static_filename + + namelist%lndp_layout = lndp_layout + namelist%lndp_input_file = lndp_input_file + namelist%lndp_output_file = lndp_output_file + + n_var_lndp= 0 + do k =1,size(lndp_var_list) + if (trim(lndp_var_list(k)) .EQ. 'XXX') then + cycle + else + n_var_lndp=n_var_lndp+1 + namelist%lndp_var_list(n_var_lndp) = lndp_var_list(k) + endif + enddo + namelist%n_var_lndp = n_var_lndp + if (n_var_lndp > max_n_var_lndp) then + print*, 'ERROR: land perturbation requested for too many parameters', & + 'increase max_n_var_lndp' + stop 10 + endif + + end subroutine ReadNamelist + +end module namelist_mod diff --git a/tile2tile/pull_request_template.md b/tile2tile/pull_request_template.md new file mode 100644 index 00000000..8eae0621 --- /dev/null +++ b/tile2tile/pull_request_template.md @@ -0,0 +1,32 @@ +PR Instructions: +1. Provide details under all headings below. +2. Assign Clara and one other person as reviewers. +3. If the PR is not ready for merging, add the "DRAFT/DO NOT MERGE" label. +4. When a PR is ready to merge, remove the "DRAFT/DO NOT MERGE" and email Clara. +5. Before requesting that the PR be merged, complete the checklist below. + +Notes on preparing PR, using git can be found in README_git + +## Describe your changes +Summarise all code changes included in PR: + +List any associated PRs in the submodules. + + +## Issue ticket number and link +List the git Issue that this PR addresses: + + +## Test output +Is this PR expected to pass the DA_IMS_test (ie., does it change the output)? + +Does it pass the DA_IMS_test? + +If changes to the test results are expected, what are these changes? Provide a link to the output directory when running the test: + +## Checklist before requesting a review +- [ ] My branch being merged is up to date with the latest develop. +- [ ] I have performed a self-review of my code by examining the differences that will be merged. +- [ ] I have not made any unnecessary code changes / changed any default behavior. +- [ ] My code passes the DA_IMS_test, or differences can be explained. + diff --git a/tile2tile/tile2tile_driver.f90 b/tile2tile/tile2tile_driver.f90 new file mode 100644 index 00000000..69a24d44 --- /dev/null +++ b/tile2tile/tile2tile_driver.f90 @@ -0,0 +1,48 @@ +program tile2tile_driver + + use namelist_mod + use tile2tile_restart_mod + implicit none + + type(namelist_type) :: namelist + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Get namelist file name from command line +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call get_command_argument(1, namelist%namelist_name) + if(namelist%namelist_name == "") then + print *, "add namelist to the command line: " + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read namelist information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadNamelist(namelist) + + print*, "conversion direction: ",namelist%direction + + if(namelist%direction /= "ufs2jedi" .and. namelist%direction /= "jedi2ufs") then + print*, "conversion direction: ",namelist%direction, " not recognized" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Decide the pathway +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + select case (namelist%direction) + + case ("ufs2jedi", "jedi2ufs") + + write(*,*) "Option: "//trim(namelist%direction) + call tile2tile_restart(namelist) + + case default + + write(*,*) "choose a valid conversion direction" + + end select +end program tile2tile_driver diff --git a/tile2tile/tile2tile_restart_mod.f90 b/tile2tile/tile2tile_restart_mod.f90 new file mode 100644 index 00000000..0a984f39 --- /dev/null +++ b/tile2tile/tile2tile_restart_mod.f90 @@ -0,0 +1,637 @@ +module tile2tile_restart_mod + + use namelist_mod + use netcdf + implicit none + + type JEDI_tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type JEDI_tile_type + + type UFS_tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type UFS_tile_type + +contains + + subroutine tile2tile_restart(namelist) + type(namelist_type) :: namelist + type(JEDI_tile_type):: JEDI_tile + type(UFS_tile_type) :: UFS_tile + character*300 :: tile_filename + character*300 :: JEDI_tile_filename + character*300 :: UFS_tile_filename + character*19 :: date + integer :: yyyy,mm,dd,hh,nn,ss + integer :: itile, ix, iy, iloc + integer :: ncid, dimid, varid, status + logical :: file_exists + read(namelist%restart_date( 1: 4),'(i4.4)') yyyy + read(namelist%restart_date( 6: 7),'(i2.2)') mm + read(namelist%restart_date( 9:10),'(i2.2)') dd + read(namelist%restart_date(12:13),'(i2.2)') hh + read(namelist%restart_date(15:16),'(i2.2)') nn + read(namelist%restart_date(18:19),'(i2.2)') ss + + write(date,'(i4,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2)') & + yyyy, "-", mm, "-", dd, "_", hh, "-", nn, "-", ss + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(JEDI_tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(JEDI_tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%soil_moisture_total (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%soil_moisture_liquid (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + + allocate(UFS_tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(UFS_tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%soil_moisture_total(namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%soil_moisture_liquid(namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a5,i1,a3)') ".tile", itile, ".nc" + + tile_filename = trim(namelist%tile_path)//trim(namelist%tile_fstub)//trim(adjustl(tile_filename)) + inquire(file=trim(tile_filename), exist=file_exists) + + print*, "Reading tile file: ", trim(tile_filename) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist1" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(trim(tile_filename), NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , JEDI_tile%land_frac(:,:,itile)) + status = nf90_get_var(ncid, varid , UFS_tile%land_frac(:,:,itile)) + + status = nf90_close(ncid) + + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Direction of transfer branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(namelist%direction == "ufs2jedi") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read UFS NoahMP restart file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call UFS_ReadTileRestart(namelist, date, UFS_tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! explicitly initialize to 0. + JEDI_tile%slmsk=0. + + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(UFS_tile%land_frac(ix,iy,itile) > 0.0) then + JEDI_tile%swe(ix,iy,itile) = UFS_tile%swe(ix,iy,itile) + JEDI_tile%vegetation_type(ix,iy,itile) = UFS_tile%vegetation_type(ix,iy,itile) + JEDI_tile%snow_depth(ix,iy,itile) = UFS_tile%snow_depth(ix,iy,itile) + JEDI_tile%active_snow_layers(ix,iy,itile) = UFS_tile%active_snow_layers(ix,iy,itile) + JEDI_tile%swe_previous(ix,iy,itile) = UFS_tile%swe_previous(ix,iy,itile) + JEDI_tile%snow_soil_interface(ix,iy,:,itile) = UFS_tile%snow_soil_interface(ix,iy,:,itile) + JEDI_tile%temperature_snow(ix,iy,:,itile) = UFS_tile%temperature_snow(ix,iy,:,itile) + JEDI_tile%snow_ice_layer(ix,iy,:,itile) = UFS_tile%snow_ice_layer(ix,iy,:,itile) + JEDI_tile%snow_liq_layer(ix,iy,:,itile) = UFS_tile%snow_liq_layer(ix,iy,:,itile) + JEDI_tile%temperature_soil(ix,iy,:,itile) = UFS_tile%temperature_soil(ix,iy,:,itile) + JEDI_tile%soil_moisture_total(ix,iy,:,itile) = UFS_tile%soil_moisture_total(ix,iy,:,itile) + JEDI_tile%slmsk(ix,iy,itile) = 1. + JEDI_tile%soil_moisture_liquid(ix,iy,:,itile)= UFS_tile%soil_moisture_liquid(ix,iy,:,itile) + JEDI_tile%temperature_ground(ix,iy,itile) = UFS_tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write JEDI tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call JEDI_WriteTileRestart(namelist, date, JEDI_tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! JEDI2UFS branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +! elseif(namelist%direction == "tile2vector") then +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!! Read JEDI tile restart files +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! call ReadTileRestart(namelist, date, JEDI_tile) +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!! Transfer tile to vector +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! iloc = 0 +! do itile = 1, 6 +! do iy = 1, namelist%tile_size +! do ix = 1, namelist%tile_size +! +! if(tile%land_frac(ix,iy,itile) > 0.0) then +! iloc = iloc + 1 +! vector%swe(iloc) = tile%swe(ix,iy,itile) +! vector%snow_depth(iloc) = tile%snow_depth(ix,iy,itile) +! vector%active_snow_layers(iloc) = tile%active_snow_layers(ix,iy,itile) +! vector%swe_previous(iloc) = tile%swe_previous(ix,iy,itile) +! vector%snow_soil_interface(iloc,:) = tile%snow_soil_interface(ix,iy,:,itile) +! vector%temperature_snow(iloc,:) = tile%temperature_snow(ix,iy,:,itile) +! vector%snow_ice_layer(iloc,:) = tile%snow_ice_layer(ix,iy,:,itile) +! vector%snow_liq_layer(iloc,:) = tile%snow_liq_layer(ix,iy,:,itile) +! vector%temperature_soil(iloc,:) = tile%temperature_soil(ix,iy,:,itile) +! vector%soil_moisture_total(iloc,:) = tile%soil_moisture_total(ix,iy,:,itile) +! vector%soil_moisture_liquid(iloc,:)= tile%soil_moisture_liquid(ix,iy,:,itile) +! vector%temperature_ground(iloc) = tile%temperature_ground(ix,iy,itile) +! end if +! +! end do +! end do +! end do +! +! print*, "Transferred ",iloc, "land grids" +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!! Write FV3 tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! call WriteVectorRestart(namelist, date, vector, vector_length) +! + end if ! "vector2tile" or "tile2vector" branch + + end subroutine tile2tile_restart + + + + + + subroutine UFS_ReadTileRestart(namelist, date, UFS_tile) + + use netcdf + + type(namelist_type) :: namelist + type(UFS_tile_type) :: UFS_tile + character*19 :: date + character*256 :: UFS_tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(UFS_tile_filename,'(a17,a19,a5,i1,a3)') & + "ufs_land_restart.", date, ".tile",itile,".nc" +! date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + UFS_tile_filename = trim(namelist%tile_restart_path)//trim(UFS_tile_filename) + + print*, "Reading UFS tile file: ", trim(UFS_tile_filename) + + inquire(file=UFS_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(UFS_tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the UFS NoahMP tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(UFS_tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading restart file + + status = nf90_inq_varid(ncid, "weasd", varid) + if (status /= nf90_noerr) then + print *, 'weasd variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_soil(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vegtype", varid) + if (status /= nf90_noerr) then + print *, 'vegtype variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine UFS_ReadTileRestart + + + subroutine JEDI_WriteTileRestart(namelist, date, JEDI_tile) + + use netcdf + + type(namelist_type) :: namelist + type(JEDI_tile_type):: JEDI_tile + character*19 :: date + character*256 :: JEDI_tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow, dim_id_snso, dim_id_time + + do itile = 1, 6 + + write(JEDI_tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + JEDI_tile_filename = trim(namelist%output_path)//trim(JEDI_tile_filename) + + print*, "Writing tile file: ", trim(JEDI_tile_filename) + + status = nf90_create(JEDI_tile_filename, NF90_CLOBBER, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid, "xaxis_1" , namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "yaxis_1" , namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_2" , 4 , dim_id_soil) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_3" , 3 , dim_id_snow) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_4" , 7 , dim_id_snso) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "Time" , NF90_UNLIMITED , dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! define dimension variables (for JEDI) + + status = nf90_def_var(ncid, "Time", NF90_DOUBLE, & + (/dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "xaxis_1", NF90_DOUBLE, & + (/dim_id_xdim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "yaxis_1", NF90_DOUBLE, & + (/dim_id_ydim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_2", NF90_DOUBLE, & + (/dim_id_soil/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_3", NF90_DOUBLE, & + (/dim_id_snow/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_4", NF90_DOUBLE, & + (/dim_id_snso/), varid) + if (status /= nf90_noerr) call handle_err(status) + + +! Define variables in the file. + + status = nf90_def_var(ncid, "sheleg", NF90_DOUBLE, & ! note: this is weasd in vector file. + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snwdph", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snowxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "sneqvoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zsnsoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snso,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tsnoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snicexy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snliqxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "stc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "smc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slmsk", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "vtype", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tgxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_enddef(ncid) + + +! fill dimension variables + + status = nf90_inq_varid(ncid, "Time", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/1/) ) + + status = nf90_inq_varid(ncid, "xaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "yaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "zaxis_2", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 4)/) ) + + status = nf90_inq_varid(ncid, "zaxis_3", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 3)/) ) + + status = nf90_inq_varid(ncid, "zaxis_4", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 7)/) ) + +! Start writing restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_soil(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + +! include in output, so can be used to id which tile grid cells are being simulated + status = nf90_inq_varid(ncid, "slmsk", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%slmsk(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + +! include for JEDI QC of SMAP obs + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine JEDI_WriteTileRestart + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module tile2tile_restart_mod diff --git a/tile2tile/user_build_config b/tile2tile/user_build_config new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/tile2tile/user_build_config @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff From 7e3dc450b200ade9c77557e0a82281f6d2629d75 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:09:07 +0000 Subject: [PATCH 09/65] update submit_cycle.sh --- do_submit_cycle.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh index 1954b551..0a805ec4 100755 --- a/do_submit_cycle.sh +++ b/do_submit_cycle.sh @@ -75,10 +75,10 @@ fi ############################ # set executables -if [[ -e ${BUILDDIR}/bin/vector2tile_converter.exe ]]; then #prefer cmake-built executables - export vec2tileexec=${BUILDDIR}/bin/vector2tile_converter.exe +if [[ -e ${BUILDDIR}/bin/tile2tile_converter.exe ]]; then #prefer cmake-built executables + export tile2tileexec=${BUILDDIR}/bin/tile2tile_converter.exe else - export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe + export tile2tileexec=${CYCLEDIR}/tile2tile/tile2tile_converter.exe fi if [[ -e ${BUILDDIR}/bin/ufsLand.exe ]]; then export LSMexec=${BUILDDIR}/bin/ufsLand.exe @@ -154,10 +154,12 @@ fi ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output # copy ICS into restarts, if needed -rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc -rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc +for tile in 1 2 3 4 5 6 +do +rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc +rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc # if restart not in experiment out directory, copy the restarts from the ICSDIR -if [[ ! -e ${rst_out} ]]; then + if [[ ! -e ${rst_out} ]]; then echo "Looking for ICS: ${rst_in}" if [[ -e ${rst_in} ]]; then echo "ICS found, copying" @@ -166,7 +168,8 @@ if [[ ! -e ${rst_out} ]]; then echo "ICS not found. Exiting" exit 10 fi -fi + fi +done # create dates file touch analdates.sh From 6fcd4d1cd1600efd3c4558aeece6e475c85b2745 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:12:25 +0000 Subject: [PATCH 10/65] update DA_update --- DA_update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DA_update b/DA_update index e4171946..21fe3816 160000 --- a/DA_update +++ b/DA_update @@ -1 +1 @@ -Subproject commit e4171946e2dd6bb1829a8bba9198486df02815bb +Subproject commit 21fe38162c800f8d0c9632accef19846763850a3 From 26786911b6db74e3ed1938033279798c70b54169 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 21 Sep 2023 14:12:47 +0000 Subject: [PATCH 11/65] update submit_cycle.sh --- submit_cycle.sh | 66 +++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index c041aaa1..5855932a 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=offline_noahmp -#SBATCH --account=da-cpu +#SBATCH --account=epic #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --tasks-per-node=6 @@ -58,7 +58,7 @@ while [ $date_count -lt $cycles_per_job ]; do nHH=`echo $NEXTDATE | cut -c9-10` ############################ - # copy restarts to workdir, convert to vector for DA (all members) + # copy restarts to workdir, convert to UFS tile for DA (all members) mem_ens="mem000" @@ -67,46 +67,47 @@ while [ $date_count -lt $cycles_per_job ]; do cd $MEM_WORKDIR # copy restarts into work directory - rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + + for tile in 1 2 3 4 5 6 + do + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc cp ${rst_in} ${rst_out} + done if [[ $do_jedi == "YES" ]]; then - echo '************************************************' - echo 'calling tile2vector' - export MEM_WORKDIR - # update vec2tile and tile2vec namelists - cp ${CYCLEDIR}/template.vector2tile vector2tile.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist - sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist - sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist - sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist - sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist - sed -i -e "s/XXRES/${RES}/g" vector2tile.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" vector2tile.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" vector2tile.namelist - - # submit vec2tile + # update tile2tile namelist + cp ${CYCLEDIR}/template.tile2tile tile2tile.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2tile.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" tile2tile.namelist + sed -i -e "s/XXMM/${MM}/g" tile2tile.namelist + sed -i -e "s/XXDD/${DD}/g" tile2tile.namelist + sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist + sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" tile2tile.namelist + sed -i -e "s/XXRES/${RES}/g" tile2tile.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2tile.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" tile2tile.namelist + + # submit tile2tile echo '************************************************' - echo 'calling vector2tile' - + echo 'calling tile2tile' + if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods fi - $vec2tileexec vector2tile.namelist + $tile2tileexec tile2tile.namelist if [[ $? != 0 ]]; then - echo "vec2tile failed" + echo "tile2tile failed" exit fi - fi # vector2tile for DA + fi # tile2tile for DA ############################ # do DA update @@ -126,9 +127,10 @@ while [ $date_count -lt $cycles_per_job ]; do exit fi fi - + + exit ############################ - # convert back to vector, run model (all members) convert back to vector, run model (all members) + # convert back to UFS tile, run model (all members) mem_ens="mem000" @@ -139,7 +141,7 @@ while [ $date_count -lt $cycles_per_job ]; do if [[ $do_jedi == "YES" ]]; then echo '************************************************' - echo 'calling tile2vector' + echo 'calling tile2tile' if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods From 55e2b89ff45a8a8e8322ec4dbabed3de27192073 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:54:16 -0400 Subject: [PATCH 12/65] Update Jenkinsfile adding sonarqube --- test/ci/Jenkinsfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index d1e456c7..4e15b74b 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -1,6 +1,16 @@ pipeline { agent none stages { + stage('Launch SonarQube') { + steps { + script { + build job: '/land-da/land-da-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: '') + ], wait: false + } + } + } stage('Land DA workflow') { agent { label 'built-in' From fd3c5cb1deccbab360c91f8d2d0e11713a42edda Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 24 Oct 2023 17:27:57 +0000 Subject: [PATCH 13/65] add jedi2ufs conversion feature --- tile2tile/tile2tile_restart_mod.f90 | 334 ++++++++++++++++++++++++---- 1 file changed, 295 insertions(+), 39 deletions(-) diff --git a/tile2tile/tile2tile_restart_mod.f90 b/tile2tile/tile2tile_restart_mod.f90 index 0a984f39..993260d5 100644 --- a/tile2tile/tile2tile_restart_mod.f90 +++ b/tile2tile/tile2tile_restart_mod.f90 @@ -187,59 +187,56 @@ subroutine tile2tile_restart(namelist) ! JEDI2UFS branch !!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! elseif(namelist%direction == "tile2vector") then -! + elseif(namelist%direction == "jedi2ufs") then + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! Read JEDI tile restart files !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! -! call ReadTileRestart(namelist, date, JEDI_tile) + call JEDI_ReadTileRestart(namelist, date, JEDI_tile) ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! Transfer tile to vector +! Transfer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! -! iloc = 0 -! do itile = 1, 6 -! do iy = 1, namelist%tile_size -! do ix = 1, namelist%tile_size -! -! if(tile%land_frac(ix,iy,itile) > 0.0) then -! iloc = iloc + 1 -! vector%swe(iloc) = tile%swe(ix,iy,itile) -! vector%snow_depth(iloc) = tile%snow_depth(ix,iy,itile) -! vector%active_snow_layers(iloc) = tile%active_snow_layers(ix,iy,itile) -! vector%swe_previous(iloc) = tile%swe_previous(ix,iy,itile) -! vector%snow_soil_interface(iloc,:) = tile%snow_soil_interface(ix,iy,:,itile) -! vector%temperature_snow(iloc,:) = tile%temperature_snow(ix,iy,:,itile) -! vector%snow_ice_layer(iloc,:) = tile%snow_ice_layer(ix,iy,:,itile) -! vector%snow_liq_layer(iloc,:) = tile%snow_liq_layer(ix,iy,:,itile) -! vector%temperature_soil(iloc,:) = tile%temperature_soil(ix,iy,:,itile) -! vector%soil_moisture_total(iloc,:) = tile%soil_moisture_total(ix,iy,:,itile) -! vector%soil_moisture_liquid(iloc,:)= tile%soil_moisture_liquid(ix,iy,:,itile) -! vector%temperature_ground(iloc) = tile%temperature_ground(ix,iy,itile) -! end if -! -! end do -! end do -! end do -! -! print*, "Transferred ",iloc, "land grids" -! + +! explicitly initialize to 0. + UFS_tile%slmsk=0. + + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(JEDI_tile%land_frac(ix,iy,itile) > 0.0) then + UFS_tile%swe(ix,iy,itile) = JEDI_tile%swe(ix,iy,itile) + UFS_tile%vegetation_type(ix,iy,itile) = JEDI_tile%vegetation_type(ix,iy,itile) + UFS_tile%snow_depth(ix,iy,itile) = JEDI_tile%snow_depth(ix,iy,itile) + UFS_tile%active_snow_layers(ix,iy,itile) = JEDI_tile%active_snow_layers(ix,iy,itile) + UFS_tile%swe_previous(ix,iy,itile) = JEDI_tile%swe_previous(ix,iy,itile) + UFS_tile%snow_soil_interface(ix,iy,:,itile) = JEDI_tile%snow_soil_interface(ix,iy,:,itile) + UFS_tile%temperature_snow(ix,iy,:,itile) = JEDI_tile%temperature_snow(ix,iy,:,itile) + UFS_tile%snow_ice_layer(ix,iy,:,itile) = JEDI_tile%snow_ice_layer(ix,iy,:,itile) + UFS_tile%snow_liq_layer(ix,iy,:,itile) = JEDI_tile%snow_liq_layer(ix,iy,:,itile) + UFS_tile%temperature_soil(ix,iy,:,itile) = JEDI_tile%temperature_soil(ix,iy,:,itile) + UFS_tile%soil_moisture_total(ix,iy,:,itile) = JEDI_tile%soil_moisture_total(ix,iy,:,itile) + UFS_tile%slmsk(ix,iy,itile) = 1. + UFS_tile%soil_moisture_liquid(ix,iy,:,itile)= JEDI_tile%soil_moisture_liquid(ix,iy,:,itile) + UFS_tile%temperature_ground(ix,iy,itile) = JEDI_tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! Write FV3 tile file +! Write UFS tile file !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! -! call WriteVectorRestart(namelist, date, vector, vector_length) + call UFS_WriteTileRestart(namelist, date, UFS_tile) ! - end if ! "vector2tile" or "tile2vector" branch + end if ! "ufs2jedi" or "jedi2ufs" branch end subroutine tile2tile_restart - - - - subroutine UFS_ReadTileRestart(namelist, date, UFS_tile) use netcdf @@ -624,6 +621,265 @@ subroutine JEDI_WriteTileRestart(namelist, date, JEDI_tile) end subroutine JEDI_WriteTileRestart + + subroutine JEDI_ReadTileRestart(namelist, date, JEDI_tile) + + use netcdf + + type(namelist_type) :: namelist + type(JEDI_tile_type) :: JEDI_tile + character*19 :: date + character*256 :: JEDI_tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + + write(JEDI_tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13),"0000.sfc_data.tile",itile,".nc" + + JEDI_tile_filename = trim(namelist%output_path)//trim(JEDI_tile_filename) + + print*, "Reading JEDI tile file: ", trim(JEDI_tile_filename) + + inquire(file=JEDI_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(JEDI_tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the JEDI tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(JEDI_tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading updated file + + status = nf90_inq_varid(ncid, "sheleg", varid) + if (status /= nf90_noerr) then + print *, 'sheleg variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%active_snow_layers(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid ,JEDI_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_snow(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_soil(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%soil_moisture_total(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid ,JEDI_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_ground(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + if (status /= nf90_noerr) then + print *, 'vegtype variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%vegetation_type(:,:,itile) ,& + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine JEDI_ReadTileRestart + + subroutine UFS_WriteTileRestart(namelist, date, UFS_tile) + + use netcdf + + type(namelist_type) :: namelist + type(UFS_tile_type):: UFS_tile + character*19 :: date + character*256 :: UFS_tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow,dim_id_snso, dim_id_time + logical :: file_exists + + do itile = 1, 6 + + write(UFS_tile_filename,'(a17,a19,a5,i1,a3)') & + "ufs_land_restart.", date, ".tile",itile,".nc" + + UFS_tile_filename = trim(namelist%output_path)//trim(UFS_tile_filename) + + inquire(file=UFS_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(UFS_tile_filename), " does not exist5" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Writing tile file: ", trim(UFS_tile_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write the tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(UFS_tile_filename, NF90_WRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "weasd", varid) + status = nf90_put_var(ncid, varid , UFS_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%active_snow_layers(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid ,UFS_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_snow(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_ice_layer(:,:,:,itile) ,& + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_liq_layer(:,:,:,itile) ,& + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_soil(:,:,:,itile), & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4,1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid ,UFS_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4,1/)) + + status = nf90_inq_varid(ncid, "vegtype", varid) + status = nf90_put_var(ncid, varid , UFS_tile%vegetation_type(:,:,itile) ,& + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , UFS_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_ground(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + end subroutine UFS_WriteTileRestart + subroutine handle_err(status) use netcdf integer, intent ( in) :: status From 1ba1ff53ff3ce11abcb3ca8a4d63993c0f65be9e Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 24 Oct 2023 17:28:55 +0000 Subject: [PATCH 14/65] add ufs2jedi and jedi2ufs tile conversion --- submit_cycle.sh | 60 ++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 5855932a..70bef86d 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -82,18 +82,18 @@ while [ $date_count -lt $cycles_per_job ]; do export MEM_WORKDIR # update tile2tile namelist - cp ${CYCLEDIR}/template.tile2tile tile2tile.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2tile.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" tile2tile.namelist - sed -i -e "s/XXMM/${MM}/g" tile2tile.namelist - sed -i -e "s/XXDD/${DD}/g" tile2tile.namelist - sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist - sed -i -e "s/XXHH/${HH}/g" tile2tile.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" tile2tile.namelist - sed -i -e "s/XXRES/${RES}/g" tile2tile.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2tile.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" tile2tile.namelist + cp ${CYCLEDIR}/template.ufs2jedi ufs2jedi.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist + sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist + sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" ufs2jedi.namelist + sed -i -e "s/XXRES/${RES}/g" ufs2jedi.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist # submit tile2tile echo '************************************************' @@ -102,7 +102,7 @@ while [ $date_count -lt $cycles_per_job ]; do if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods fi - $tile2tileexec tile2tile.namelist + $tile2tileexec ufs2jedi.namelist if [[ $? != 0 ]]; then echo "tile2tile failed" exit @@ -128,7 +128,6 @@ while [ $date_count -lt $cycles_per_job ]; do fi fi - exit ############################ # convert back to UFS tile, run model (all members) @@ -147,28 +146,33 @@ while [ $date_count -lt $cycles_per_job ]; do source ${CYCLEDIR}/land_mods fi - cp ${CYCLEDIR}/template.tile2vector tile2vector.namelist + cp ${CYCLEDIR}/template.jedi2ufs jedi2ufs.namelist - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist - sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist - sed -i -e "s/XXDD/${DD}/g" tile2vector.namelist - sed -i -e "s/XXHH/${HH}/g" tile2vector.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist - sed -i -e "s/XXRES/${RES}/g" tile2vector.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2vector.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" tile2vector.namelist - - $vec2tileexec tile2vector.namelist + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist + sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist + sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist + sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" jedi2ufs.namelist + sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist + + $tile2tileexec jedi2ufs.namelist if [[ $? != 0 ]]; then - echo "tile2vector failed" + echo "tile2tile failed" exit fi # save analysis restart - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + done + fi + exit ############################ # run the forecast model set -x From 8a56f978a570129adb8b0d924cbf7e3481e07798 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 24 Oct 2023 17:32:00 +0000 Subject: [PATCH 15/65] remove unnecessary templates --- template.tile2tile | 52 -------------------------------------------- template.tile2vector | 52 -------------------------------------------- 2 files changed, 104 deletions(-) delete mode 100644 template.tile2tile delete mode 100644 template.tile2vector diff --git a/template.tile2tile b/template.tile2tile deleted file mode 100644 index 084195be..00000000 --- a/template.tile2tile +++ /dev/null @@ -1,52 +0,0 @@ -&run_setup - -!------------------- common ------------------- -! Direction of conversion: either "vector2tile" or "tile2vector" for restart file -! "lndp2tile" or "lndp2vector" for perturbation - - direction = "ufs2jedi" - -! FV3 resolution and path to oro files for restart/perturbation conversion - - tile_size = XXRES - tile_path = "XXTPATH" - tile_fstub = "XXTSTUB" - -!------------------- only restart conversion ------------------- -! Time stamp for conversion for restart conversion - - restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" - -! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" - -! Location of vector restart file (vector2tile direction) - - vector_restart_path = "./" - -! Location of tile restart files (tile2vector direction) - -! tile_restart_path = "junk" - tile_restart_path = "./" -! Path for converted files; if same as tile/vector path, files may be overwritten - - output_path = "./" - -!------------------- only perturbation mapping ------------------- -! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file - - lndp_layout = "" - -! input perturbation pattern files - - lndp_input_file = "" - -! output files - - lndp_output_file = "" - -! land perturbation variable list, for an example: 'vgf','smc' - - lndp_var_list = '' - -/ diff --git a/template.tile2vector b/template.tile2vector deleted file mode 100644 index 4655a86d..00000000 --- a/template.tile2vector +++ /dev/null @@ -1,52 +0,0 @@ -&run_setup - -!------------------- common ------------------- -! Direction of conversion: either "vector2tile" or "tile2vector" for restart file -! "lndp2tile" or "lndp2vector" for perturbation - - direction = "tile2vector" - -! FV3 resolution and path to oro files for restart/perturbation conversion - - tile_size = XXRES - tile_path = "XXTPATH" - tile_fstub = "XXTSTUB" - -!------------------- only restart conversion ------------------- -! Time stamp for conversion for restart conversion - - restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" - -! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" - -! Location of vector restart file (vector2tile direction) - - vector_restart_path = "junk" - -! Location of tile restart files (tile2vector direction) - - tile_restart_path = "./" - -! Path for converted files; if same as tile/vector path, files may be overwritten - - output_path = "./" - -!------------------- only perturbation mapping ------------------- -! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file - - lndp_layout = "" - -! input perturbation pattern files - - lndp_input_file = "" - -! output files - - lndp_output_file = "" - -! land perturbation variable list, for an example: 'vgf','smc' - - lndp_var_list = '' - -/ From 6f79b505a79df709be8198756396f59bab540e32 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 24 Oct 2023 17:32:28 +0000 Subject: [PATCH 16/65] add new tile2tile templates --- template.jedi2ufs | 52 +++++++++++++++++++++++++++++++++++++++++++++++ template.ufs2jedi | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 template.jedi2ufs create mode 100644 template.ufs2jedi diff --git a/template.jedi2ufs b/template.jedi2ufs new file mode 100644 index 00000000..08cd7ffd --- /dev/null +++ b/template.jedi2ufs @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "ufs2jedi" or "jedi2ufs" for restart file +! + + direction = "jedi2ufs" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + +! tile_restart_path = "junk" + tile_restart_path = "./" +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ diff --git a/template.ufs2jedi b/template.ufs2jedi new file mode 100644 index 00000000..b5028419 --- /dev/null +++ b/template.ufs2jedi @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "ufs2jedi" or "jedi2ufs" for restart file +! + + direction = "ufs2jedi" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + +! tile_restart_path = "junk" + tile_restart_path = "./" +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ From 04b7afaab9c7b93f6524c5d5c5a28e98d5029311 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Wed, 25 Oct 2023 21:32:49 +0000 Subject: [PATCH 17/65] turn on the ufs-weather-model --- CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8641ea8..841873c0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,15 +44,15 @@ include_directories( ${NETCDF_INCLUDE_DIRS} ) ################################################################################ set( LANDDA_LINKER_LANGUAGE CXX) -#ExternalProject_Add(ufs-weather-model -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model -# GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model -# GIT_SUBMODULES_RECURSE TRUE -# GIT_TAG develop -# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model -# CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release -# INSTALL_COMMAND cmake -# ) +ExternalProject_Add(ufs-weather-model + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ufs-weather-model + GIT_REPOSITORY https://github.com/ufs-community/ufs-weather-model + GIT_SUBMODULES_RECURSE TRUE + GIT_TAG develop + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model + CMAKE_ARGS ${SOURCE_DIR} -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release + INSTALL_COMMAND cmake + ) add_subdirectory( tile2tile ) add_subdirectory( ufs-land-driver-emc-dev ) From 77e9c4b7e7c7fb5969bee327d7c2f891285a77f7 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 2 Nov 2023 17:30:54 +0000 Subject: [PATCH 18/65] add the UFS Noah MP run feature --- submit_cycle.sh | 132 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 27 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 70bef86d..f2b33261 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -172,37 +172,115 @@ while [ $date_count -lt $cycles_per_job ]; do fi - exit ############################ # run the forecast model - set -x - # update model namelist - cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist - sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist - sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist - sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist - sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist - sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist - sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist - - # submit model - echo '************************************************' - echo "calling model" - echo $MEM_WORKDIR + set -e - nt=$SLURM_NTASKS - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${CYCLEDIR}/land_mods - fi + echo '************************************************' + echo 'running the forecast model' - if [[ $BASELINE =~ 'hera.internal' ]]; then - srun '--export=ALL' --label -K -n $nt $LSMexec - else - ${MPIEXEC} -n 1 $LSMexec - fi + export MACHINE_ID=${MACHINE_ID:-hera} + TEST_NAME=datm_cdeps_lnd_gswp3 + TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst + PATHRT=${CYCLEDIR}/ufs-weather-model/tests + RT_COMPILER=${RT_COMPILER:-intel} + ATOL="1e-7" + source ${PATHRT}/detect_machine.sh + source ${PATHRT}/rt_utils.sh + source ${PATHRT}/default_vars.sh + source ${PATHRT}/tests/$TEST_NAME + source ${PATHRT}/atparse.bash + + # Set inputdata location for each machines + echo "MACHINE_ID: $MACHINE_ID" + if [[ $MACHINE_ID = orion ]]; then + DISKNM=/work/noaa/nems/emc.nemspara/RT + elif [[ $MACHINE_ID = hera ]]; then + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + else + echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" + fi + + source ${PATHRT}/bl_date.conf + #BL_DATE=20230815 + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} + + echo "RTPWD= $RTPWD" + echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" + + if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 + fi + + # create test folder + RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME} + [[ -d ${RUNDIR} ]] && echo "Warning: remove old test folder!" && rm -rf ${RUNDIR} + mkdir -p ${RUNDIR} + cd ${RUNDIR} + + echo "NoahMP run dir= $RUNDIR" + + # modify some env variables - reduce core usage + export ATM_compute_tasks=0 + export ATM_io_tasks=1 + export LND_tasks=6 + export layout_x=1 + export layout_y=1 + + # FV3 executable: + cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model + + #set multiple input files + for i in ${FV3_RUN:-fv3_run.IN} + do + atparse < ${PATHRT}/fv3_conf/${i} >> fv3_run # Need to Update for warm start option, R.K, 11/2/2023 + done + + if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then + if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then + atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml + fi + fi + + atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure + + compute_petbounds_and_tasks + + atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure + + # diag table + if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then + atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table + fi + + # Field table + if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then + cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table + fi + + # Field Dictionary + cp ${PATHRT}/parm/fd_nems.yaml fd_nems.yaml + + # Set up the run directory + source ./fv3_run + + if [[ $DATM_CDEPS = 'true' ]]; then + atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in} > datm_in + atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams + fi + + # NoahMP table file + cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl + + # start runs + echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" + export MPIRUN=${MPIRUN:-`which mpiexec`} + ${MPIRUN} -n ${TASKS} ./ufs_model + + exit + # no error codes on exit from model, check for restart below instead ############################ From 518c8f674dea4d41722bd11c17538a6dc9f9f21e Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 2 Nov 2023 17:32:19 +0000 Subject: [PATCH 19/65] increse cpus per task from 1 to 2 --- submit_cycle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index f2b33261..912d2cbd 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -4,7 +4,7 @@ #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --tasks-per-node=6 -#SBATCH --cpus-per-task=1 +#SBATCH --cpus-per-task=2 #SBATCH -t 00:06:00 #SBATCH -o log_noahmp.%j.log #SBATCH -e err_noahmp.%j.err From 4436a2d60a491185f8472bad28c082c6592a16ac Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 6 Nov 2023 21:44:09 +0000 Subject: [PATCH 20/65] update the location of restart files in 'fv3_run' --- submit_cycle.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 912d2cbd..0de4706e 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -171,7 +171,7 @@ while [ $date_count -lt $cycles_per_job ]; do done fi - + ############################ # run the forecast model set -e @@ -188,7 +188,7 @@ while [ $date_count -lt $cycles_per_job ]; do source ${PATHRT}/detect_machine.sh source ${PATHRT}/rt_utils.sh source ${PATHRT}/default_vars.sh - source ${PATHRT}/tests/$TEST_NAME + source ${PATHRT}/tests/$TEST_NAME_RST source ${PATHRT}/atparse.bash # Set inputdata location for each machines @@ -214,9 +214,9 @@ while [ $date_count -lt $cycles_per_job ]; do exit 1 fi - # create test folder - RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME} - [[ -d ${RUNDIR} ]] && echo "Warning: remove old test folder!" && rm -rf ${RUNDIR} + # create run folder + RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} + [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} mkdir -p ${RUNDIR} cd ${RUNDIR} @@ -233,11 +233,13 @@ while [ $date_count -lt $cycles_per_job ]; do cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model #set multiple input files - for i in ${FV3_RUN:-fv3_run.IN} - do - atparse < ${PATHRT}/fv3_conf/${i} >> fv3_run # Need to Update for warm start option, R.K, 11/2/2023 - done + #for i in ${FV3_RUN:-fv3_run.IN} + #do + # atparse < ${PATHRT}/fv3_conf/${i} >> fv3_run # Need to Update for warm start option, R.K, 11/2/2023 + #done + cp ${LANDDA_INPUTS}/restarts/fv3_run ./fv3_run + if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml From 78f498a63ba21c1bb53b2e7bac918552178aff1f Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 6 Nov 2023 21:44:58 +0000 Subject: [PATCH 21/65] increase cpus per tasks from 2 to 5 --- submit_cycle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 0de4706e..38a8fc64 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -4,7 +4,7 @@ #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --tasks-per-node=6 -#SBATCH --cpus-per-task=2 +#SBATCH --cpus-per-task=5 #SBATCH -t 00:06:00 #SBATCH -o log_noahmp.%j.log #SBATCH -e err_noahmp.%j.err From 6d4528a50d0d46bb07766c6f8969a94e944c5b8c Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 9 Nov 2023 17:11:34 +0000 Subject: [PATCH 22/65] add vector2tile submodule again --- .gitmodules | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitmodules b/.gitmodules index 949965c6..edf1e642 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,7 @@ path = tile2tile url = https://github.com/rhaesung/land-tile2tile.git branch = feature/porting +[submodule "vector2tile"] + path = vector2tile + url = https://github.com/NOAA-PSL/land-vector2tile.git + branch = develop From 2dc77ebecf7eca217748b796b5d3581fdef9416a Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 9 Nov 2023 17:26:37 +0000 Subject: [PATCH 23/65] update CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 841873c0..49a26108 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ ExternalProject_Add(ufs-weather-model INSTALL_COMMAND cmake ) +add_subdirectory( vector2tile ) add_subdirectory( tile2tile ) add_subdirectory( ufs-land-driver-emc-dev ) add_subdirectory( DA_update/add_jedi_incr ) From 92afb174b3052a8e355617a1873a04002164ee8a Mon Sep 17 00:00:00 2001 From: rhaesung Date: Thu, 9 Nov 2023 18:15:08 +0000 Subject: [PATCH 24/65] add vec2tile again --- vector2tile/.gitignore | 4 + vector2tile/CMakeLists.txt | 76 ++ vector2tile/Makefile | 24 + vector2tile/README.md | 30 + .../cmake/compiler_flags_GNU_CXX.cmake | 45 + .../cmake/compiler_flags_GNU_Fortran.cmake | 43 + .../compiler_flags_GNU_Fortran.cmake_test | 45 + .../cmake/compiler_flags_Intel_Fortran.cmake | 41 + .../cmake/vector2tile_compiler_flags.cmake | 23 + .../config/user_build_config.gfortran.serial | 11 + vector2tile/config/user_build_config.hera | 10 + vector2tile/configure | 39 + vector2tile/namelist.lndp2vector | 52 + vector2tile/namelist.vector2tile | 52 + vector2tile/namelist_mod.f90 | 86 ++ vector2tile/pull_request_template.md | 32 + vector2tile/user_build_config | 10 + vector2tile/vector2tile_driver.f90 | 55 ++ vector2tile/vector2tile_perturbation_mod.f90 | 347 +++++++ vector2tile/vector2tile_restart_mod.f90 | 929 ++++++++++++++++++ 20 files changed, 1954 insertions(+) create mode 100644 vector2tile/.gitignore create mode 100644 vector2tile/CMakeLists.txt create mode 100644 vector2tile/Makefile create mode 100644 vector2tile/README.md create mode 100755 vector2tile/cmake/compiler_flags_GNU_CXX.cmake create mode 100755 vector2tile/cmake/compiler_flags_GNU_Fortran.cmake create mode 100755 vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test create mode 100755 vector2tile/cmake/compiler_flags_Intel_Fortran.cmake create mode 100755 vector2tile/cmake/vector2tile_compiler_flags.cmake create mode 100644 vector2tile/config/user_build_config.gfortran.serial create mode 100644 vector2tile/config/user_build_config.hera create mode 100755 vector2tile/configure create mode 100644 vector2tile/namelist.lndp2vector create mode 100644 vector2tile/namelist.vector2tile create mode 100644 vector2tile/namelist_mod.f90 create mode 100644 vector2tile/pull_request_template.md create mode 100644 vector2tile/user_build_config create mode 100644 vector2tile/vector2tile_driver.f90 create mode 100644 vector2tile/vector2tile_perturbation_mod.f90 create mode 100644 vector2tile/vector2tile_restart_mod.f90 diff --git a/vector2tile/.gitignore b/vector2tile/.gitignore new file mode 100644 index 00000000..18de9433 --- /dev/null +++ b/vector2tile/.gitignore @@ -0,0 +1,4 @@ +vector2tile_converter.exe +vector2tile_converter.o +*.mod +*.o diff --git a/vector2tile/CMakeLists.txt b/vector2tile/CMakeLists.txt new file mode 100644 index 00000000..81ac1f91 --- /dev/null +++ b/vector2tile/CMakeLists.txt @@ -0,0 +1,76 @@ +# (C) Copyright 2022 . +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +cmake_minimum_required( VERSION 3.12) +project( vector2tile VERSION 2022.10 LANGUAGES Fortran ) + +# ecbuild integration +find_package(ecbuild 3.3.2 REQUIRED) +include( ecbuild_system NO_POLICY_SCOPE ) +ecbuild_declare_project() + +list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +include( vector2tile_compiler_flags ) + +################################################################################ +# Dependencies +################################################################################ + +find_package(OpenMP COMPONENTS C Fortran) +find_package(MPI REQUIRED COMPONENTS C Fortran) +find_package(NetCDF REQUIRED COMPONENTS Fortran ) + +################################################################################ +# Sources +################################################################################ + +set( VECTOR2TILE_LINKER_LANGUAGE Fortran ) + +list ( APPEND src_files +./namelist_mod.f90 +./vector2tile_perturbation_mod.f90 +./vector2tile_restart_mod.f90 +) + +ecbuild_add_library( TARGET vector2tile + SOURCES ${src_files} + INSTALL_HEADERS LISTED + LINKER_LANGUAGE ${VECTOR2TILE_LINKER_LANGUAGE} + ) + + +target_link_libraries(vector2tile PUBLIC NetCDF::NetCDF_Fortran) +target_link_libraries(vector2tile PUBLIC MPI::MPI_Fortran) +target_link_libraries(vector2tile PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_Fortran) + +# Fortran module output directory for build and install interfaces +set(MODULE_DIR module/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}) +set_target_properties(${PROJECT_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}) +install(DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}/ DESTINATION ${MODULE_DIR}) +target_include_directories(${PROJECT_NAME} INTERFACE + $ + $) + +set( VECTOR2TILE_INCLUDE_DIRS ${CMAKE_Fortran_MODULE_DIRECTORY} ) +set( VECTOR2TILE_LIBRARIES vector2tile ) + +# vector2tile_converter.exe executable +#------------------------------------------------------------------------------- +set ( exe_files vector2tile_driver.f90 ) + +ecbuild_add_executable( TARGET vector2tile_converter.exe + SOURCES ${exe_files} + LIBS vector2tile + LINKER_LANGUAGE ${VECTOR2TILE_LINKER_LANGUAGE} + ) + + +################################################################################ +# Finalise configuration +################################################################################ + +ecbuild_install_project( NAME vector2tile ) +ecbuild_print_summary() diff --git a/vector2tile/Makefile b/vector2tile/Makefile new file mode 100644 index 00000000..45308c7e --- /dev/null +++ b/vector2tile/Makefile @@ -0,0 +1,24 @@ +# Makefile +# +.SUFFIXES: +.SUFFIXES: .o .f90 + +include ./user_build_config + +OBJS = namelist_mod.o vector2tile_restart_mod.o vector2tile_perturbation_mod.o vector2tile_driver.o + +all: vector2tile_converter.exe + +.f90.o: + $(COMPILERF90) -c $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(*).f90 + +vector2tile_converter.exe: $(OBJS) + $(COMPILERF90) -o $(@) $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(OBJS) $(NETCDFLIB) + +clean: + rm -f *.o *.mod *.exe + + +# +# Dependencies: +# diff --git a/vector2tile/README.md b/vector2tile/README.md new file mode 100644 index 00000000..69c46d60 --- /dev/null +++ b/vector2tile/README.md @@ -0,0 +1,30 @@ +Code to map between vector format used by the Noah-MP offline driver, and the tile format used by the UFS atmospheric model. Currently used to prepare input tile files for JEDI. These files include only those fields required by JEDI, rather than the full restart. Can also be used to map stochastic_physics output to the tile or vector. Stochastic physics output files depend on the layout, with 1x1 layout giving one file per tile. + +Mike Barlage, Clara Draper. Dec 2021. + +To compile on hera: + +>configure + + choose hera + + load the modules indicated + +>make + +To run: + +>vector2tile_converter.exe namelist.vector2tile + +the namelist defines the conversion direction and the paths of the files + +Details: + +the vector2tile pathway assumes that the vector file exists in the vector_restart_path directory and overwrites/creates tile files in the output_path + +the tile2vector pathway is a little tricky, it assumes the tile files exist in tile_restart_path and overwrites only the snow variables in the vector file in the output_path. If the vector file does not exist in output_path the process will fail. + +the overall assumption here is that we will have a full model vector restart file, then convert the vector to tiles for only snow variables, then convert the updated snow variables back to the full vector restart file + +For the lndp2vector or lndp2tile option, a new file will be created with the pertbations and lat lon only. + diff --git a/vector2tile/cmake/compiler_flags_GNU_CXX.cmake b/vector2tile/cmake/compiler_flags_GNU_CXX.cmake new file mode 100755 index 00000000..dbe13c02 --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_CXX.cmake @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +if( HAVE_OMP ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fopenmp") +else( ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fno-openmp") +endif( ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_DEBUG "-O0 -g" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_BIT "-O2" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_CXX_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake new file mode 100755 index 00000000..c0b618cc --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake @@ -0,0 +1,43 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test new file mode 100755 index 00000000..95f6209b --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -fbacktrace -ffree-line-length-none -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -ffree-form") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) +set( CMAKE_Fortran_FLAGS_RELEASE " " ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake b/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake new file mode 100755 index 00000000..7eef4052 --- /dev/null +++ b/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake @@ -0,0 +1,41 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -convert big_endian -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -sox -traceback -msse2" ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3 -debug minimal -fp-model source" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -debug minimal -fp-model source" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/vector2tile/cmake/vector2tile_compiler_flags.cmake b/vector2tile/cmake/vector2tile_compiler_flags.cmake new file mode 100755 index 00000000..8aebfaec --- /dev/null +++ b/vector2tile/cmake/vector2tile_compiler_flags.cmake @@ -0,0 +1,23 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#add_definitions ( -Duse_libMPI -Duse_netCDF -DSPMD -DNXGLOB=4) +add_definitions ( -Duse_libMPI -Duse_netCDF -Dncdf -Dgather_scatter_barrier) + +####################################################################################### +# Fortran +####################################################################################### + +if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) + include( compiler_flags_GNU_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) + include( compiler_flags_Intel_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "XL" ) + include( compiler_flags_XL_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" ) + include( compiler_flags_Cray_Fortran ) +else() + message( STATUS "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options") +endif() diff --git a/vector2tile/config/user_build_config.gfortran.serial b/vector2tile/config/user_build_config.gfortran.serial new file mode 100644 index 00000000..a7191d5c --- /dev/null +++ b/vector2tile/config/user_build_config.gfortran.serial @@ -0,0 +1,11 @@ + +#=============================================================================== +# Placeholder options for Mac gfortran +#=============================================================================== + + COMPILERF90 = /opt/local/bin/gfortran-mp-11 + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/opt/local/include + NETCDFLIB = -L/opt/local/lib -lnetcdf -lnetcdff + diff --git a/vector2tile/config/user_build_config.hera b/vector2tile/config/user_build_config.hera new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/vector2tile/config/user_build_config.hera @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/vector2tile/configure b/vector2tile/configure new file mode 100755 index 00000000..24aad730 --- /dev/null +++ b/vector2tile/configure @@ -0,0 +1,39 @@ +#!/usr/bin/perl + + if($#ARGV == 0) { + $response = shift(@ARGV) ; + print("Configure soil test environment: $response \n"); + }else { + print "Please select from following predefined environments: \n\n"; + + print " 1. hera \n"; + print " 2. orion \n"; + print " 3. gfortran compiler serial \n"; + print " 0. exit only \n"; + + printf "\nEnter selection : " ; + + $response = ; + chop($response); + } + + if ($response == 1) { + # Hera settings + system "cp config/user_build_config.hera user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load 2022.2.0 netcdf/4.7.0 \n\n\n"; + } + + elsif ($response == 2) { + # Orion settings + system "cp config/user_build_config.orion user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load intel netcdf \n\n\n"; + } + + elsif ($response == 3) { + # Mike's Mac settings + system "cp config/user_build_config.gfortran.serial user_build_config"; + } + + else {print "no selection $response\n"; last} diff --git a/vector2tile/namelist.lndp2vector b/vector2tile/namelist.lndp2vector new file mode 100644 index 00000000..92ade568 --- /dev/null +++ b/vector2tile/namelist.lndp2vector @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "lndp2vector" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "junk" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "junk" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x1, 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x1" + +! input files + + lndp_input_file = "./stochy_out1/workg_T162_984x488.tile01.nc" + +! output files + + lndp_output_file = "./output_ens001.nc" + +! land perturbation variable list + + lndp_var_list='vgf' + +/ diff --git a/vector2tile/namelist.vector2tile b/vector2tile/namelist.vector2tile new file mode 100644 index 00000000..a7a0b680 --- /dev/null +++ b/vector2tile/namelist.vector2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "vector2tile" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/restart/" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x4" + +! input files + + lndp_input_file = "/scratch2/NCEPDEV/land/data/DA/ensemble_pert/workg_T162_984x488.tileXX.nc" + +! output files + + lndp_output_file = "./output.nc" + +! land perturbation variable list + + lndp_var_list='vgf','smc' + +/ diff --git a/vector2tile/namelist_mod.f90 b/vector2tile/namelist_mod.f90 new file mode 100644 index 00000000..4fbca14b --- /dev/null +++ b/vector2tile/namelist_mod.f90 @@ -0,0 +1,86 @@ +module namelist_mod + implicit none + + integer, parameter :: max_n_var_lndp = 20 + type namelist_type + character*256 :: namelist_name = "" + character*11 :: direction = "" + character*256 :: tile_path = "" + character*256 :: tile_fstub = "" + integer :: tile_size + character*19 :: restart_date = "" + character*256 :: vector_restart_path = "" + character*256 :: tile_restart_path = "" + character*256 :: output_path = "" + character*256 :: static_filename = "" + character*3 :: lndp_layout = "" + character*256 :: lndp_input_file = "" + character*256 :: lndp_output_file = "" + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + end type namelist_type + +contains + + subroutine ReadNamelist(namelist) + + type(namelist_type) :: namelist + character*11 :: direction + character*256 :: tile_path + character*256 :: tile_fstub + integer :: tile_size + character*19 :: restart_date + character*256 :: vector_restart_path + character*256 :: tile_restart_path + character*256 :: output_path + character*256 :: static_filename + character*3 :: lndp_layout + character*256 :: lndp_input_file + character*256 :: lndp_output_file + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + integer :: k + + namelist / run_setup / direction, tile_path, tile_fstub, tile_size, restart_date, vector_restart_path, & + tile_restart_path, output_path, static_filename, lndp_layout, & + lndp_input_file, lndp_output_file, lndp_var_list, n_var_lndp + + lndp_var_list = 'XXX' + + open(30, file=namelist%namelist_name, form="formatted") + read(30, run_setup) + close(30) + + namelist%direction = direction + namelist%tile_path = tile_path + namelist%tile_fstub = tile_fstub + namelist%tile_size = tile_size + namelist%restart_date = restart_date + namelist%vector_restart_path = vector_restart_path + namelist%tile_restart_path = tile_restart_path + namelist%output_path = output_path + namelist%static_filename = static_filename + + namelist%lndp_layout = lndp_layout + namelist%lndp_input_file = lndp_input_file + namelist%lndp_output_file = lndp_output_file + + n_var_lndp= 0 + do k =1,size(lndp_var_list) + if (trim(lndp_var_list(k)) .EQ. 'XXX') then + cycle + else + n_var_lndp=n_var_lndp+1 + namelist%lndp_var_list(n_var_lndp) = lndp_var_list(k) + endif + enddo + namelist%n_var_lndp = n_var_lndp + if (n_var_lndp > max_n_var_lndp) then + print*, 'ERROR: land perturbation requested for too many parameters', & + 'increase max_n_var_lndp' + stop 10 + endif + + end subroutine ReadNamelist + +end module namelist_mod diff --git a/vector2tile/pull_request_template.md b/vector2tile/pull_request_template.md new file mode 100644 index 00000000..8eae0621 --- /dev/null +++ b/vector2tile/pull_request_template.md @@ -0,0 +1,32 @@ +PR Instructions: +1. Provide details under all headings below. +2. Assign Clara and one other person as reviewers. +3. If the PR is not ready for merging, add the "DRAFT/DO NOT MERGE" label. +4. When a PR is ready to merge, remove the "DRAFT/DO NOT MERGE" and email Clara. +5. Before requesting that the PR be merged, complete the checklist below. + +Notes on preparing PR, using git can be found in README_git + +## Describe your changes +Summarise all code changes included in PR: + +List any associated PRs in the submodules. + + +## Issue ticket number and link +List the git Issue that this PR addresses: + + +## Test output +Is this PR expected to pass the DA_IMS_test (ie., does it change the output)? + +Does it pass the DA_IMS_test? + +If changes to the test results are expected, what are these changes? Provide a link to the output directory when running the test: + +## Checklist before requesting a review +- [ ] My branch being merged is up to date with the latest develop. +- [ ] I have performed a self-review of my code by examining the differences that will be merged. +- [ ] I have not made any unnecessary code changes / changed any default behavior. +- [ ] My code passes the DA_IMS_test, or differences can be explained. + diff --git a/vector2tile/user_build_config b/vector2tile/user_build_config new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/vector2tile/user_build_config @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/vector2tile/vector2tile_driver.f90 b/vector2tile/vector2tile_driver.f90 new file mode 100644 index 00000000..988d49d3 --- /dev/null +++ b/vector2tile/vector2tile_driver.f90 @@ -0,0 +1,55 @@ +program vector2tile_driver + + use namelist_mod + use vector2tile_restart_mod + use vector2tile_perturbation_mod + implicit none + + type(namelist_type) :: namelist + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Get namelist file name from command line +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call get_command_argument(1, namelist%namelist_name) + if(namelist%namelist_name == "") then + print *, "add namelist to the command line: " + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read namelist information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadNamelist(namelist) + + print*, "conversion direction: ",namelist%direction + + if(namelist%direction /= "tile2vector" .and. namelist%direction /= "vector2tile" .and. & + namelist%direction /= "lndp2vector" .and. namelist%direction /= "lndp2tile") then + print*, "conversion direction: ",namelist%direction, " not recognized" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Decide the pathway +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + select case (namelist%direction) + + case ("tile2vector", "vector2tile") + + write(*,*) "Option: "//trim(namelist%direction) + call vector2tile_restart(namelist) + + case ("lndp2vector", "lndp2tile") + + write(*,*) "Option: "//trim(namelist%direction) + call mapping_perturbation(namelist) + + case default + + write(*,*) "choose a valid conversion direction" + + end select +end program vector2tile_driver diff --git a/vector2tile/vector2tile_perturbation_mod.f90 b/vector2tile/vector2tile_perturbation_mod.f90 new file mode 100644 index 00000000..1b0f977a --- /dev/null +++ b/vector2tile/vector2tile_perturbation_mod.f90 @@ -0,0 +1,347 @@ +module vector2tile_perturbation_mod + + use netcdf + use namelist_mod + implicit none + +contains + + subroutine mapping_perturbation(namelist) + type(namelist_type) :: namelist + real, allocatable :: var_vector(:), lon_vector(:), lat_vector(:) + real, allocatable :: var_tile(:,:,:), lon_tile(:,:,:), lat_tile(:,:,:) + real, allocatable :: land_frac_tile(:,:,:) + character*256 :: vector_filename + character*256 :: tile_filename + character*256 :: input_filename + character*256 :: output_filename + character*2 :: tile1, tile2 + real, allocatable :: tmp2d(:,:) + integer :: filename_length + integer :: vector_length = 0 + integer :: layout_x, layout_y, nx, ny + integer :: itile, ix, iy, iloc, ivar + integer :: i, j, m, n, i1, i2, j1, j2, t2 + integer :: ncid, dimid, varid, status + integer :: dim_id_xdim, dim_id_ydim, dim_id_time + integer :: ncid_landp, ncid_vec, ncid_tile(6) + logical :: file_exists + + if(trim(namelist%lndp_layout) == '1x4') then + layout_x = 1 + layout_y = 4 + else if(trim(namelist%lndp_layout) == '2x2') then + layout_x = 2 + layout_y = 2 + else if(trim(namelist%lndp_layout) == '4x1') then + layout_x = 4 + layout_y = 1 + else if(trim(namelist%lndp_layout) == '1x1') then + layout_x = 1 + layout_y = 1 + else + print*, "layout: ",namelist%lndp_layout, " not recognized" + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + allocate(var_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(lon_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(lat_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(land_frac_tile(namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + + if(namelist%tile_size < 100) then + write(tile_filename,'(a5,i2,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + elseif(namelist%tile_size < 1000) then + write(tile_filename,'(a5,i3,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + elseif(namelist%tile_size < 10000) then + write(tile_filename,'(a5,i4,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + else + print *, "unknown tile size" + stop 10 + end if + + tile_filename = trim(namelist%tile_path)//trim(tile_filename) + + inquire(file=tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist" + print*, "In routine mapping_perturbation: check paths and file name" + stop 10 + end if + + status = nf90_open(tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , land_frac_tile(:,:,itile)) + + status = nf90_close(ncid) + + vector_length = vector_length + count(land_frac_tile(:,:,itile) > 0) + + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate vector variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(var_vector(vector_length)) + allocate(lon_vector(vector_length)) + allocate(lat_vector(vector_length)) + + nx = namelist%tile_size/layout_x + ny = namelist%tile_size/layout_y + + if(namelist%n_var_lndp > 0) then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the output file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if(trim(namelist%direction) == "lndp2vector") then + + output_filename = namelist%lndp_output_file + status = nf90_create(output_filename, NF90_CLOBBER, ncid_vec) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid_vec, "location", vector_length, dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_vec, "Time", NF90_UNLIMITED, dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! Define lat and lon + status = nf90_def_var(ncid_vec, 'lon_vec', & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_var(ncid_vec, 'lat_vec', & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + +! Define variables in the file. + do ivar = 1, namelist%n_var_lndp + status = nf90_def_var(ncid_vec, namelist%lndp_var_list(ivar), & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + enddo + status = nf90_enddef(ncid_vec) + + else if(trim(namelist%direction) == "lndp2tile" ) then + + filename_length = len_trim(namelist%lndp_output_file) + do itile = 1, 6 + write(tile1,fmt='(I2.2)') itile + output_filename = trim(namelist%lndp_output_file(1:filename_length-5))//trim(tile1)//'.nc' + status = nf90_create(output_filename, NF90_CLOBBER, ncid_tile(itile)) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid_tile(itile), "xaxis_1", namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_tile(itile), "yaxis_1", namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_tile(itile), "Time", NF90_UNLIMITED, dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! Define lat and lon in the file + status = nf90_def_var(ncid_tile(itile), 'lon_tile', & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_var(ncid_tile(itile), 'lat_tile', & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + +! Define variables in the file. + do ivar = 1, namelist%n_var_lndp + status = nf90_def_var(ncid_tile(itile), namelist%lndp_var_list(ivar), & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + enddo + status = nf90_enddef(ncid_tile(itile)) + enddo + endif + + allocate(tmp2d(nx,ny)) + + do ivar = 1, namelist%n_var_lndp + t2 = 1 + filename_length = len_trim(namelist%lndp_input_file) + do itile = 1, 6 + i1=1 + i2=i1+nx-1 + j1=1 + j2=j1+ny-1 + do j=1,layout_y + do i=1,layout_x + write(tile2,fmt='(I2.2)') t2 + if(t2 > 1) then + i1=i1+nx + i2=i2+nx + if (i2 .GT. namelist%tile_size) then + i1=1 + i2=i1+nx-1 + endif + endif + input_filename = trim(namelist%lndp_input_file(1:filename_length-5))//trim(tile2)//'.nc' + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Read the perturbation pattern + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(input_filename, NF90_NOWRITE, ncid_landp) + status = nf90_inq_varid(ncid_landp, namelist%lndp_var_list(ivar), varid) + if (status /= nf90_noerr) then + print *, trim(namelist%lndp_var_list(ivar))//' variable missing from perturbation file' + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + var_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + ! get the lat and lon + if ( ivar == 1 ) then + status = nf90_inq_varid(ncid_landp, 'grid_lon', varid) + if (status /= nf90_noerr) then + print *, trim('grid lon variable missing from perturbation file') + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + lon_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + ! get the lat and lat + status = nf90_inq_varid(ncid_landp, 'grid_lat', varid) + if (status /= nf90_noerr) then + print *, trim('grid lat variable missing from perturbation file') + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + lat_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + endif + + t2 = t2+1 + + enddo + + j1=j1+ny + j2=j2+ny + + if (j2 .GT. namelist%tile_size) then + j1=1 + j2=j1+ny-1 + endif + + enddo + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Write the perturbation pattern for the tile files + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2tile") then + status = nf90_inq_varid(ncid_tile(itile), namelist%lndp_var_list(ivar), varid) + status = nf90_put_var(ncid_tile(itile), varid , var_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + if (ivar==1) then + status = nf90_inq_varid(ncid_tile(itile), 'lon_grid', varid) + status = nf90_put_var(ncid_tile(itile), varid , lon_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + status = nf90_inq_varid(ncid_tile(itile), 'lat_grid', varid) + status = nf90_put_var(ncid_tile(itile), varid , lat_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + endif + endif + + enddo ! for each tile + + + iloc = 0 + do itile = 1, 6 + do j = 1, namelist%tile_size + do i = 1, namelist%tile_size + if(land_frac_tile(i,j,itile) > 0.0) then + iloc = iloc + 1 + var_vector(iloc) = var_tile(i,j,itile) + if (ivar==1) then + lon_vector(iloc) = lon_tile(i,j,itile) + lat_vector(iloc) = lat_tile(i,j,itile) + endif + endif + enddo + enddo + enddo ! for each tile + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Write the perturbation pattern for the vector file + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2vector") then + status = nf90_inq_varid(ncid_vec, namelist%lndp_var_list(ivar), varid) + status = nf90_put_var(ncid_vec, varid , var_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + + if (ivar == 1) then + status = nf90_inq_varid(ncid_vec, 'lon_vec', varid) + status = nf90_put_var(ncid_vec, varid , lon_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + status = nf90_inq_varid(ncid_vec, 'lat_vec', varid) + status = nf90_put_var(ncid_vec, varid , lat_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + endif + endif + + enddo ! for each variable + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Close the netcdf file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2vector") then + status = nf90_close(ncid_vec) + else if(trim(namelist%direction) == "lndp2tile") then + do itile = 1, 6 + status = nf90_close(ncid_tile(itile)) + enddo + endif + endif + end subroutine mapping_perturbation + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module vector2tile_perturbation_mod diff --git a/vector2tile/vector2tile_restart_mod.f90 b/vector2tile/vector2tile_restart_mod.f90 new file mode 100644 index 00000000..1679294c --- /dev/null +++ b/vector2tile/vector2tile_restart_mod.f90 @@ -0,0 +1,929 @@ +module vector2tile_restart_mod + + use namelist_mod + use netcdf + implicit none + + type vector_type + double precision, allocatable :: swe (:) + double precision, allocatable :: snow_depth (:) + double precision, allocatable :: active_snow_layers (:) + double precision, allocatable :: swe_previous (:) + double precision, allocatable :: snow_soil_interface(:,:) + double precision, allocatable :: temperature_snow (:,:) + double precision, allocatable :: snow_ice_layer (:,:) + double precision, allocatable :: snow_liq_layer (:,:) + double precision, allocatable :: temperature_soil (:,:) +! needed for IMSaggregate_mod + double precision, allocatable :: vegetation_type(:) +! needed by JEDI to mask out land-ice + double precision, allocatable :: soil_moisture_total(:,:) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid(:,:) + double precision, allocatable :: temperature_ground (:) + end type vector_type + + type tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type tile_type + +contains + + subroutine vector2tile_restart(namelist) + type(namelist_type) :: namelist + type(vector_type) :: vector + type(tile_type) :: tile + character*256 :: vector_filename + character*300 :: tile_filename + character*19 :: date + integer :: vector_length = 0 + integer :: yyyy,mm,dd,hh,nn,ss + integer :: itile, ix, iy, iloc + integer :: ncid, dimid, varid, status + logical :: file_exists + read(namelist%restart_date( 1: 4),'(i4.4)') yyyy + read(namelist%restart_date( 6: 7),'(i2.2)') mm + read(namelist%restart_date( 9:10),'(i2.2)') dd + read(namelist%restart_date(12:13),'(i2.2)') hh + read(namelist%restart_date(15:16),'(i2.2)') nn + read(namelist%restart_date(18:19),'(i2.2)') ss + + write(date,'(i4,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2)') & + yyyy, "-", mm, "-", dd, "_", hh, "-", nn, "-", ss + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%soil_moisture_total (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%soil_moisture_liquid (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a5,i1,a3)') ".tile", itile, ".nc" + + tile_filename = trim(namelist%tile_path)//trim(namelist%tile_fstub)//trim(adjustl(tile_filename)) + inquire(file=trim(tile_filename), exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist1" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(trim(tile_filename), NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , tile%land_frac(:,:,itile)) + + status = nf90_close(ncid) + + vector_length = vector_length + count(tile%land_frac(:,:,itile) > 0) + + end do + + print*, "The FV3 tiles report ",vector_length, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate vector variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(vector%swe (vector_length)) + allocate(vector%snow_depth (vector_length)) + allocate(vector%active_snow_layers (vector_length)) + allocate(vector%swe_previous (vector_length)) + allocate(vector%snow_soil_interface(vector_length,7)) + allocate(vector%temperature_snow (vector_length,3)) + allocate(vector%snow_ice_layer (vector_length,3)) + allocate(vector%snow_liq_layer (vector_length,3)) + allocate(vector%temperature_soil (vector_length,4)) + allocate(vector%soil_moisture_total (vector_length,4)) + allocate(vector%vegetation_type (vector_length)) + allocate(vector%soil_moisture_liquid (vector_length,4)) + allocate(vector%temperature_ground (vector_length)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Direction of transfer branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(namelist%direction == "vector2tile") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read vector restart file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorRestart(namelist, date, vector, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer vector to tiles +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! explicitly initialize to 0. + tile%slmsk=0. + + iloc = 0 + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + if(tile%land_frac(ix,iy,itile) > 0.0) then + iloc = iloc + 1 + tile%swe(ix,iy,itile) = vector%swe(iloc) + tile%vegetation_type(ix,iy,itile) = vector%vegetation_type(iloc) + tile%snow_depth(ix,iy,itile) = vector%snow_depth(iloc) + tile%active_snow_layers(ix,iy,itile) = vector%active_snow_layers(iloc) + tile%swe_previous(ix,iy,itile) = vector%swe_previous(iloc) + tile%snow_soil_interface(ix,iy,:,itile) = vector%snow_soil_interface(iloc,:) + tile%temperature_snow(ix,iy,:,itile) = vector%temperature_snow(iloc,:) + tile%snow_ice_layer(ix,iy,:,itile) = vector%snow_ice_layer(iloc,:) + tile%snow_liq_layer(ix,iy,:,itile) = vector%snow_liq_layer(iloc,:) + tile%temperature_soil(ix,iy,:,itile) = vector%temperature_soil(iloc,:) + tile%soil_moisture_total(ix,iy,:,itile) = vector%soil_moisture_total(iloc,:) + tile%slmsk(ix,iy,itile) = 1. + tile%soil_moisture_liquid(ix,iy,:,itile)= vector%soil_moisture_liquid(iloc,:) + tile%temperature_ground(ix,iy,itile) = vector%temperature_ground(iloc) + end if + + end do + end do + end do + + print*, "Transferred ",iloc, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write FV3 tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call WriteTileRestart(namelist, date, tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! tile2vector branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + elseif(namelist%direction == "tile2vector") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read tile restart files +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadTileRestart(namelist, date, tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer tile to vector +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + iloc = 0 + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(tile%land_frac(ix,iy,itile) > 0.0) then + iloc = iloc + 1 + vector%swe(iloc) = tile%swe(ix,iy,itile) + vector%snow_depth(iloc) = tile%snow_depth(ix,iy,itile) + vector%active_snow_layers(iloc) = tile%active_snow_layers(ix,iy,itile) + vector%swe_previous(iloc) = tile%swe_previous(ix,iy,itile) + vector%snow_soil_interface(iloc,:) = tile%snow_soil_interface(ix,iy,:,itile) + vector%temperature_snow(iloc,:) = tile%temperature_snow(ix,iy,:,itile) + vector%snow_ice_layer(iloc,:) = tile%snow_ice_layer(ix,iy,:,itile) + vector%snow_liq_layer(iloc,:) = tile%snow_liq_layer(ix,iy,:,itile) + vector%temperature_soil(iloc,:) = tile%temperature_soil(ix,iy,:,itile) + vector%soil_moisture_total(iloc,:) = tile%soil_moisture_total(ix,iy,:,itile) + vector%soil_moisture_liquid(iloc,:)= tile%soil_moisture_liquid(ix,iy,:,itile) + vector%temperature_ground(iloc) = tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + + print*, "Transferred ",iloc, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write FV3 tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call WriteVectorRestart(namelist, date, vector, vector_length) + + end if ! "vector2tile" or "tile2vector" branch + + end subroutine vector2tile_restart + + subroutine ReadVectorRestart(namelist, date, vector, vector_length) + + use netcdf + + type(namelist_type) :: namelist + type(vector_type) :: vector + character*19 :: date + integer :: vector_length + character*256 :: vector_filename, filename + integer :: ncid, dimid, varid, status + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create vector file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + write(vector_filename,'(a17,a19,a3)') "ufs_land_restart.", date, ".nc" + + filename = trim(namelist%vector_restart_path)//trim(vector_filename) + + inquire(file=filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(filename), " does not exist2" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Reading vector file: ", trim(filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Check the vector length, fail if not consistent with tile-calculated length +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorLength(filename, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the vector fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_varid(ncid, "snow_water_equiv", varid) + if (status /= nf90_noerr) then + print *, 'snow_water_equiv variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%swe , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_depth", varid) + if (status /= nf90_noerr) then + print *, 'snow_depth variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_depth , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "active_snow_levels", varid) + if (status /= nf90_noerr) then + print *, 'active_snow_levels variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%active_snow_layers , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_water_equiv_old", varid) + if (status /= nf90_noerr) then + print *, 'snow_water_equiv_old variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%swe_previous, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "temperature_snow", varid) + if (status /= nf90_noerr) then + print *, 'temperature_snow variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_snow , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "interface_depth", varid) + if (status /= nf90_noerr) then + print *, 'interface_depth variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_soil_interface , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 7, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_ice", varid) + if (status /= nf90_noerr) then + print *, 'snow_level_ice variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_ice_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_liquid", varid) + if (status /= nf90_noerr) then + print *, 'snow_level_liquid variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_liq_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "temperature_soil", varid) + if (status /= nf90_noerr) then + print *, 'temperature_soil variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_soil , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + status = nf90_inq_varid(ncid, "soil_moisture_vol", varid) + if (status /= nf90_noerr) then + print *, 'soil_moisture_vol variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%soil_moisture_total , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + status = nf90_inq_varid(ncid, "soil_liquid_vol", varid) + if (status /= nf90_noerr) then + print *, 'soil_liquid_vol variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%soil_moisture_liquid , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + + status = nf90_inq_varid(ncid, "temperature_ground", varid) + if (status /= nf90_noerr) then + print *, 'temperature_ground variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_ground , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + + ! read vegetation from static file + + filename = trim(namelist%static_filename) + + inquire(file=filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(filename), " does not exist3" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_varid(ncid, "vegetation_category", varid) + if (status /= nf90_noerr) then + print *, 'vegetation_category missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%vegetation_type, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + end subroutine ReadVectorRestart + + subroutine ReadTileRestart(namelist, date, tile) + + use netcdf + + type(namelist_type) :: namelist + type(tile_type) :: tile + character*19 :: date + character*256 :: tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + tile_filename = trim(namelist%tile_restart_path)//trim(tile_filename) + + inquire(file=tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Reading tile file: ", trim(tile_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + if (status /= nf90_noerr) then + print *, 'sheleg variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_soil(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%soil_moisture_total(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine ReadTileRestart + + subroutine WriteVectorRestart(namelist, date, vector, vector_length) + + use netcdf + + type(namelist_type) :: namelist + type(vector_type) :: vector + character*19 :: date + integer :: vector_length + character*256 :: vector_filename + integer :: ncid, dimid, varid, status + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create vector file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + write(vector_filename,'(a17,a19,a3)') "ufs_land_restart.", date, ".nc" + + vector_filename = trim(namelist%output_path)//trim(vector_filename) + + inquire(file=vector_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(vector_filename), " does not exist5" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Writing vector file: ", trim(vector_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Check the vector length, fail if not consistent with tile-calculated length +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorLength(vector_filename, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write the vector fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(vector_filename, NF90_WRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "snow_water_equiv", varid) + status = nf90_put_var(ncid, varid , vector%swe , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_depth", varid) + status = nf90_put_var(ncid, varid , vector%snow_depth , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "active_snow_levels", varid) + status = nf90_put_var(ncid, varid , vector%active_snow_layers , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_water_equiv_old", varid) + status = nf90_put_var(ncid, varid , vector%swe_previous, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "temperature_snow", varid) + status = nf90_put_var(ncid, varid , vector%temperature_snow , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "interface_depth", varid) + status = nf90_put_var(ncid, varid , vector%snow_soil_interface , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 7, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_ice", varid) + status = nf90_put_var(ncid, varid , vector%snow_ice_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_liquid", varid) + status = nf90_put_var(ncid, varid , vector%snow_liq_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "temperature_soil", varid) + status = nf90_put_var(ncid, varid , vector%temperature_soil , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "soil_moisture_vol", varid) + status = nf90_put_var(ncid, varid , vector%soil_moisture_total , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "soil_liquid_vol", varid) + status = nf90_put_var(ncid, varid , vector%soil_moisture_liquid , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "temperature_ground", varid) + status = nf90_put_var(ncid, varid , vector%temperature_ground , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + + end subroutine WriteVectorRestart + + subroutine WriteTileRestart(namelist, date, tile) + + use netcdf + + type(namelist_type) :: namelist + type(tile_type) :: tile + character*19 :: date + character*256 :: tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow, dim_id_snso, dim_id_time + + do itile = 1, 6 + + !write(tile_filename,'(a17,a19,a5,i1,a3)') "ufs_land_restart.", date, ".tile", itile, ".nc" + write(tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + tile_filename = trim(namelist%output_path)//trim(tile_filename) + + print*, "Writing tile file: ", trim(tile_filename) + + status = nf90_create(tile_filename, NF90_CLOBBER, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid, "xaxis_1" , namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "yaxis_1" , namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_2" , 4 , dim_id_soil) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_3" , 3 , dim_id_snow) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_4" , 7 , dim_id_snso) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "Time" , NF90_UNLIMITED , dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! define dimension variables (for JEDI) + + status = nf90_def_var(ncid, "Time", NF90_DOUBLE, & + (/dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "xaxis_1", NF90_DOUBLE, & + (/dim_id_xdim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "yaxis_1", NF90_DOUBLE, & + (/dim_id_ydim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_2", NF90_DOUBLE, & + (/dim_id_soil/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_3", NF90_DOUBLE, & + (/dim_id_snow/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_4", NF90_DOUBLE, & + (/dim_id_snso/), varid) + if (status /= nf90_noerr) call handle_err(status) + + +! Define variables in the file. + + status = nf90_def_var(ncid, "sheleg", NF90_DOUBLE, & ! note: this is weasd in vector file. + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snwdph", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snowxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "sneqvoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zsnsoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snso,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tsnoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snicexy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snliqxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "stc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "smc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slmsk", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "vtype", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tgxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_enddef(ncid) + + +! fill dimension variables + + status = nf90_inq_varid(ncid, "Time", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/1/) ) + + status = nf90_inq_varid(ncid, "xaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "yaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "zaxis_2", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 4)/) ) + + status = nf90_inq_varid(ncid, "zaxis_3", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 3)/) ) + + status = nf90_inq_varid(ncid, "zaxis_4", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 7)/) ) + +! Start writing restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + status = nf90_put_var(ncid, varid , tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid , tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , tile%temperature_soil(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid , tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + +! include in output, so can be used to id which tile grid cells are being simulated + status = nf90_inq_varid(ncid, "slmsk", varid) + status = nf90_put_var(ncid, varid , tile%slmsk(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + status = nf90_put_var(ncid, varid , tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + +! include for JEDI QC of SMAP obs + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine WriteTileRestart + + subroutine ReadVectorLength(filename, vector_length) + + use netcdf + + character*256 :: filename + integer :: vector_length + integer :: length_from_file + integer :: ncid, dimid, varid, status + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_dimid(ncid, "location", dimid) + status = nf90_inquire_dimension(ncid, dimid, len = length_from_file) + + status = nf90_close(ncid) + + if(vector_length /= length_from_file) then + print*, "number of land points in tiles not consistent with land model vector length" + stop 10 + else + print*, "number of land points in tiles consistent with land model vector length" + end if + + end subroutine ReadVectorLength + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module vector2tile_restart_mod From aef338c697760153c5686eac5288fddab936837f Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 13 Nov 2023 15:02:59 +0000 Subject: [PATCH 25/65] adding the offline noah-mp feature with era5 forcing --- do_submit_cycle.sh | 54 +++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh index 0a805ec4..0ce64348 100755 --- a/do_submit_cycle.sh +++ b/do_submit_cycle.sh @@ -75,6 +75,11 @@ fi ############################ # set executables +if [[ -e ${BUILDDIR}/bin/vector2tile_converter.exe ]]; then #prefer cmake-built executables + export vec2tileexec=${BUILDDIR}/bin/vector2tile_converter.exe +else + export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe +fi if [[ -e ${BUILDDIR}/bin/tile2tile_converter.exe ]]; then #prefer cmake-built executables export tile2tileexec=${BUILDDIR}/bin/tile2tile_converter.exe else @@ -154,22 +159,41 @@ fi ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output # copy ICS into restarts, if needed -for tile in 1 2 3 4 5 6 -do -rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc -rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc -# if restart not in experiment out directory, copy the restarts from the ICSDIR - if [[ ! -e ${rst_out} ]]; then - echo "Looking for ICS: ${rst_in}" - if [[ -e ${rst_in} ]]; then - echo "ICS found, copying" - cp ${rst_in} ${rst_out} - else - echo "ICS not found. Exiting" - exit 10 +if [[ $atmos_forc == "era5" ]]; then + rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc + # if restart not in experiment out directory, copy the restarts from the ICSDIR + if [[ ! -e ${rst_out} ]]; then + echo "Looking for ICS: ${rst_in}" + if [[ -e ${rst_in} ]]; then + echo "ICS found, copying" + cp ${rst_in} ${rst_out} + else + echo "ICS not found. Exiting" + exit 10 + fi fi - fi -done +fi + +if [[ $atmos_forc == "gswp3" ]]; then + + for tile in 1 2 3 4 5 6 + do + rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${sYYYY}-${sMM}-${sDD}-00000.tile${tile}.nc + # if restart not in experiment out directory, copy the restarts from the ICSDIR + if [[ ! -e ${rst_out} ]]; then + echo "Looking for ICS: ${rst_in}" + if [[ -e ${rst_in} ]]; then + echo "ICS found, copying" + cp ${rst_in} ${rst_out} + else + echo "ICS not found. Exiting" + exit 10 + fi + fi + done +fi # create dates file touch analdates.sh From 3f3a3b2ab8853122e2c1e6361b6b51497facdd9b Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 13 Nov 2023 15:04:15 +0000 Subject: [PATCH 26/65] add vec2tile/tile2vec templates --- template.tile2vector | 52 ++++++++++++++++++++++++++++++++++++++++++++ template.vector2tile | 52 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 template.tile2vector create mode 100644 template.vector2tile diff --git a/template.tile2vector b/template.tile2vector new file mode 100644 index 00000000..4655a86d --- /dev/null +++ b/template.tile2vector @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "tile2vector" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "junk" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "./" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ diff --git a/template.vector2tile b/template.vector2tile new file mode 100644 index 00000000..e175d2e0 --- /dev/null +++ b/template.vector2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "vector2tile" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "junk" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ From 1786c936e37de67385a65dff6f3cd099dab4acc2 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 13 Nov 2023 15:05:10 +0000 Subject: [PATCH 27/65] add settings_DA_cycle_gswp3 --- settings_DA_cycle_gswp3 | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 settings_DA_cycle_gswp3 diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3 new file mode 100644 index 00000000..83d8372b --- /dev/null +++ b/settings_DA_cycle_gswp3 @@ -0,0 +1,50 @@ +# Settings file for submit_cycle, for running the DA_GHCN_test + +# experiment name +export exp_name=DA_gswp3 +#export BASELINE=hera.internal + +# experiment period +STARTDATE=2000010300 +ENDDATE=2000010400 + +# Get commmon variables +source ./release.environment +############################ + +#forcing options: gswp3, era5 +export atmos_forc=gswp3 + +# for LETKF, this is size of ensemble. +# for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 +export ensemble_size=1 + +# length of each forecast +export FCSTHR=24 + +#FV3 resolution +export RES=96 +if [[ $BASELINE =~ 'hera.internal' ]]; then + export TPATH=/scratch2/NCEPDEV/land/data/fix/C96.mx100_frac/ +else + export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/" +fi +export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH + # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean. + +# number of cycles to submit in a single job +export cycles_per_job=1 + +# directory with initial conditions +# can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single +export ICSDIR=$LANDDAROOT/inputs/forcing/${atmos_forc}/orog_files/ + +# namelist for do_landDA.sh +# set to "openloop" to not call do_landDA.sh +export DA_config="settings_DA_test" + +# if want different DA at different times, list here. +export DA_config00=${DA_config} +export DA_config06=${DA_config} +export DA_config12=${DA_config} +export DA_config18=${DA_config} From 714728cba5f1b11bbd773a492dc5dc67d69904a4 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Mon, 13 Nov 2023 15:13:00 +0000 Subject: [PATCH 28/65] add back the offline noah-mp option with era5 forcing --- submit_cycle.sh | 340 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 229 insertions(+), 111 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 38a8fc64..2ac2dd97 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -66,21 +66,69 @@ while [ $date_count -lt $cycles_per_job ]; do MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} cd $MEM_WORKDIR - # copy restarts into work directory - for tile in 1 2 3 4 5 6 - do - rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${rst_in} ${rst_out} - done + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then + + # copy restarts into work directory + rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + cp ${rst_in} ${rst_out} + + echo '************************************************' + echo 'calling vector2tile' - if [[ $do_jedi == "YES" ]]; then export MEM_WORKDIR + # update vec2tile and tile2vec namelists + cp ${CYCLEDIR}/template.vector2tile vector2tile.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist + sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist + sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist + sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist + sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist + sed -i -e "s/XXRES/${RES}/g" vector2tile.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" vector2tile.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" vector2tile.namelist + + # submit vec2tile + echo '************************************************' + echo 'calling vector2tile' + + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + $vec2tileexec vector2tile.namelist + if [[ $? != 0 ]]; then + echo "vec2tile failed" + exit + fi + fi # vector2tile for DA + + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then + + echo '************************************************' + echo 'calling tile2tile' + + export MEM_WORKDIR + + # copy restarts into work directory + + for tile in 1 2 3 4 5 6 + do + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${rst_in} ${rst_out} + done + # update tile2tile namelist cp ${CYCLEDIR}/template.ufs2jedi ufs2jedi.namelist @@ -96,8 +144,6 @@ while [ $date_count -lt $cycles_per_job ]; do sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist # submit tile2tile - echo '************************************************' - echo 'calling tile2tile' if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods @@ -129,7 +175,7 @@ while [ $date_count -lt $cycles_per_job ]; do fi ############################ - # convert back to UFS tile, run model (all members) + # convert back to vector/tile mem_ens="mem000" @@ -138,7 +184,39 @@ while [ $date_count -lt $cycles_per_job ]; do cd $MEM_WORKDIR - if [[ $do_jedi == "YES" ]]; then + # convert back to vector, run model (all members) convert back to vector, run model (all members) + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then + echo '************************************************' + echo 'calling tile2vector' + + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + + cp ${CYCLEDIR}/template.tile2vector tile2vector.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist + sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist + sed -i -e "s/XXDD/${DD}/g" tile2vector.namelist + sed -i -e "s/XXHH/${HH}/g" tile2vector.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist + sed -i -e "s/XXRES/${RES}/g" tile2vector.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2vector.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" tile2vector.namelist + + $vec2tileexec tile2vector.namelist + if [[ $? != 0 ]]; then + echo "tile2vector failed" + exit + fi + + # save analysis restart + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + fi + + # convert back to UFS tile, run model (all members) + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then echo '************************************************' echo 'calling tile2tile' @@ -168,120 +246,150 @@ while [ $date_count -lt $cycles_per_job ]; do for tile in 1 2 3 4 5 6 do cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc done - fi ############################ # run the forecast model - set -e - - echo '************************************************' - echo 'running the forecast model' - - export MACHINE_ID=${MACHINE_ID:-hera} - TEST_NAME=datm_cdeps_lnd_gswp3 - TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst - PATHRT=${CYCLEDIR}/ufs-weather-model/tests - RT_COMPILER=${RT_COMPILER:-intel} - ATOL="1e-7" - source ${PATHRT}/detect_machine.sh - source ${PATHRT}/rt_utils.sh - source ${PATHRT}/default_vars.sh - source ${PATHRT}/tests/$TEST_NAME_RST - source ${PATHRT}/atparse.bash - - # Set inputdata location for each machines - echo "MACHINE_ID: $MACHINE_ID" - if [[ $MACHINE_ID = orion ]]; then - DISKNM=/work/noaa/nems/emc.nemspara/RT - elif [[ $MACHINE_ID = hera ]]; then - DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT - else - echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" - fi - - source ${PATHRT}/bl_date.conf - #BL_DATE=20230815 - RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} - INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} - - echo "RTPWD= $RTPWD" - echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" - - if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then - echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" - exit 1 - fi - - # create run folder - RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} - [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} - mkdir -p ${RUNDIR} - cd ${RUNDIR} - - echo "NoahMP run dir= $RUNDIR" - # modify some env variables - reduce core usage - export ATM_compute_tasks=0 - export ATM_io_tasks=1 - export LND_tasks=6 - export layout_x=1 - export layout_y=1 - - # FV3 executable: - cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then + echo '************************************************' + echo 'running the forecast model' + + set -x + + # update model namelist + cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist + sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist + sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist + sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist + sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist + sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist + sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist + + # submit model + echo $MEM_WORKDIR + + nt=$SLURM_NTASKS + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + + if [[ $BASELINE =~ 'hera.internal' ]]; then + srun '--export=ALL' --label -K -n $nt $LSMexec + else + ${MPIEXEC} -n 1 $LSMexec + fi + fi + # no error codes on exit from model, check for restart below instead - #set multiple input files - #for i in ${FV3_RUN:-fv3_run.IN} - #do - # atparse < ${PATHRT}/fv3_conf/${i} >> fv3_run # Need to Update for warm start option, R.K, 11/2/2023 - #done - cp ${LANDDA_INPUTS}/restarts/fv3_run ./fv3_run + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then + set -e + + echo '************************************************' + echo 'running the forecast model' + + export MACHINE_ID=${MACHINE_ID:-hera} + TEST_NAME=datm_cdeps_lnd_gswp3 + TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst + PATHRT=${CYCLEDIR}/ufs-weather-model/tests + RT_COMPILER=${RT_COMPILER:-intel} + ATOL="1e-7" + source ${PATHRT}/detect_machine.sh + source ${PATHRT}/rt_utils.sh + source ${PATHRT}/default_vars.sh + source ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/atparse.bash + + # Set inputdata location for each machines + echo "MACHINE_ID: $MACHINE_ID" + if [[ $MACHINE_ID = orion ]]; then + DISKNM=/work/noaa/nems/emc.nemspara/RT + elif [[ $MACHINE_ID = hera ]]; then + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + else + echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" + fi + + source ${PATHRT}/bl_date.conf + #BL_DATE=20230815 + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} + + echo "RTPWD= $RTPWD" + echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" + + if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 + fi + + # create run folder + RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} + [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} + mkdir -p ${RUNDIR} + cd ${RUNDIR} + + echo "NoahMP run dir= $RUNDIR" + + # modify some env variables - reduce core usage + export ATM_compute_tasks=0 + export ATM_io_tasks=1 + export LND_tasks=6 + export layout_x=1 + export layout_y=1 + + # FV3 executable: + cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model + cp ${LANDDA_INPUTS}/restarts/fv3_run ./fv3_run - if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then - if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then - atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml - fi - fi + if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then + if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then + atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml + fi + fi - atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure + atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure - compute_petbounds_and_tasks + compute_petbounds_and_tasks - atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure + atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure - # diag table - if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then - atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table - fi + # diag table + if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then + atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table + fi - # Field table - if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then - cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table - fi + # Field table + if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then + cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table + fi - # Field Dictionary - cp ${PATHRT}/parm/fd_nems.yaml fd_nems.yaml + # Field Dictionary + cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml - # Set up the run directory - source ./fv3_run + # Set up the run directory + source ./fv3_run - if [[ $DATM_CDEPS = 'true' ]]; then - atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in} > datm_in - atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams - fi + if [[ $DATM_CDEPS = 'true' ]]; then + atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in} > datm_in + atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams + fi - # NoahMP table file - cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl + # NoahMP table file + cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl - # start runs - echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" - export MPIRUN=${MPIRUN:-`which mpiexec`} - ${MPIRUN} -n ${TASKS} ./ufs_model + # start runs + echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" + export MPIRUN=${MPIRUN:-`which mpiexec`} + ${MPIRUN} -n ${TASKS} ./ufs_model - exit + fi # no error codes on exit from model, check for restart below instead @@ -293,13 +401,23 @@ while [ $date_count -lt $cycles_per_job ]; do MEM_WORKDIR=${WORKDIR}/${mem_ens} MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then - cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc + if [[ $atmos_forc == "era5" && -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc else echo "Something is wrong, probably the model, exiting" exit fi + if [[ $atmos_forc == "gswp3" && -e ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile1.nc ]]; then + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + done + else + echo "Something is wrong, probably the model, exiting" + exit + fi + # echo "Finished job number, ${date_count},for date: ${THISDATE}" >> $logfile THISDATE=$NEXTDATE From 478d6352e2ee0f6517880995455c2da73f3db6f2 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:24:58 +0000 Subject: [PATCH 29/65] add the fake_observations --- DA_update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DA_update b/DA_update index 21fe3816..31fff859 160000 --- a/DA_update +++ b/DA_update @@ -1 +1 @@ -Subproject commit 21fe38162c800f8d0c9632accef19846763850a3 +Subproject commit 31fff859b0b1d2ce8128781b8abffa8d27e916c5 From 1b51eea943453d1dee9add54c4d07b6660d7d0e6 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:25:21 +0000 Subject: [PATCH 30/65] update the gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index edf1e642..e739a926 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ [submodule "DA_update"] path = DA_update url = https://github.com/ufs-community/land-DA.git - branch = develop + branch = feature/porting [submodule "tile2tile"] path = tile2tile url = https://github.com/rhaesung/land-tile2tile.git From d094dd81c8b7510a3ddf5c787f46de240f2f1ed4 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:26:13 +0000 Subject: [PATCH 31/65] update do_subnit_cycle.sh --- do_submit_cycle.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh index 0ce64348..a40174c8 100755 --- a/do_submit_cycle.sh +++ b/do_submit_cycle.sh @@ -20,7 +20,6 @@ source $config_file export KEEPWORKDIR="YES" - ############################ # ensure necessary envars are set envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "CYCLEDIR" \ From b1c72bd83cb6318e432e7072cba960c940bacb29 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:27:31 +0000 Subject: [PATCH 32/65] update settings for era5 --- settings_DA_cycle_era5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings_DA_cycle_era5 b/settings_DA_cycle_era5 index edf5721b..a5c68dc0 100644 --- a/settings_DA_cycle_era5 +++ b/settings_DA_cycle_era5 @@ -1,7 +1,7 @@ # Settings file for submit_cycle, for running the DA_GHCN_test # experiment name -export exp_name=DA_GHCN_test +export exp_name=DA_ERA5_test #export BASELINE=hera.internal #ERA5 forcing is available for 2020-2021 @@ -12,7 +12,7 @@ ENDDATE=2019122200 source ./release.environment ############################ -#forcing options: gdas, era5 +#forcing options: gswp3, era5 export atmos_forc=era5 # for LETKF, this is size of ensemble. From b22521097f1b2dfddfb1fc3e735b6bceac483d22 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:28:53 +0000 Subject: [PATCH 33/65] update settings for gswp3 --- settings_DA_cycle_gswp3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3 index 83d8372b..84620acd 100644 --- a/settings_DA_cycle_gswp3 +++ b/settings_DA_cycle_gswp3 @@ -1,10 +1,10 @@ # Settings file for submit_cycle, for running the DA_GHCN_test # experiment name -export exp_name=DA_gswp3 +export exp_name=DA_GSWP3_test #export BASELINE=hera.internal -# experiment period +# experiment period STARTDATE=2000010300 ENDDATE=2000010400 From 8787552adb485e85ca9e2a02939b4d1e5cf05386 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 14 Nov 2023 22:47:01 +0000 Subject: [PATCH 34/65] Read in customized UFS datm_cdeps_lnd_gswp3_rst RT script and add write option of the UFS-land restart file as background file --- submit_cycle.sh | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 2ac2dd97..5bb11900 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --job-name=offline_noahmp +#SBATCH --job-name=ufs_land_da #SBATCH --account=epic #SBATCH --qos=debug #SBATCH --nodes=1 @@ -121,7 +121,7 @@ while [ $date_count -lt $cycles_per_job ]; do for tile in 1 2 3 4 5 6 do - rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc if [[ ! -e ${rst_in} ]]; then rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc fi @@ -249,7 +249,7 @@ while [ $date_count -lt $cycles_per_job ]; do cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc done fi - + ############################ # run the forecast model @@ -287,10 +287,9 @@ while [ $date_count -lt $cycles_per_job ]; do fi # no error codes on exit from model, check for restart below instead - if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then set -e - + echo '************************************************' echo 'running the forecast model' @@ -300,7 +299,9 @@ while [ $date_count -lt $cycles_per_job ]; do PATHRT=${CYCLEDIR}/ufs-weather-model/tests RT_COMPILER=${RT_COMPILER:-intel} ATOL="1e-7" - source ${PATHRT}/detect_machine.sh + + cp ${LANDDA_INPUTS}/restarts/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/detect_machine.sh source ${PATHRT}/rt_utils.sh source ${PATHRT}/default_vars.sh source ${PATHRT}/tests/$TEST_NAME_RST @@ -388,7 +389,6 @@ while [ $date_count -lt $cycles_per_job ]; do echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" export MPIRUN=${MPIRUN:-`which mpiexec`} ${MPIRUN} -n ${TASKS} ./ufs_model - fi # no error codes on exit from model, check for restart below instead @@ -396,30 +396,21 @@ while [ $date_count -lt $cycles_per_job ]; do ############################ # check model ouput (all members) - mem_ens="mem000" + mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - - if [[ $atmos_forc == "era5" && -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then - cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc - else - echo "Something is wrong, probably the model, exiting" - exit - fi - - if [[ $atmos_forc == "gswp3" && -e ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile1.nc ]]; then - for tile in 1 2 3 4 5 6 - do - cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc - done + + if [[ $atmos_forc == "era5" ]]; then + # if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc else - echo "Something is wrong, probably the model, exiting" - exit + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + done fi - -# echo "Finished job number, ${date_count},for date: ${THISDATE}" >> $logfile - + THISDATE=$NEXTDATE date_count=$((date_count+1)) From b6eb14f506a7999a1b7807c51e71fd690b484941 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Wed, 15 Nov 2023 13:44:17 +0000 Subject: [PATCH 35/65] update file names from nems to ufs --- test/run_ufs_datm_lnd.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/run_ufs_datm_lnd.sh b/test/run_ufs_datm_lnd.sh index 6f0056a6..091d59cf 100755 --- a/test/run_ufs_datm_lnd.sh +++ b/test/run_ufs_datm_lnd.sh @@ -26,13 +26,13 @@ echo "MACHINE_ID: $MACHINE_ID" if [[ $MACHINE_ID = orion ]]; then DISKNM=/work/noaa/nems/emc.nemspara/RT elif [[ $MACHINE_ID = hera ]]; then - DISKNM=/scratch1/NCEPDEV/nems/emc.nemspara/RT + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT else echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" fi -#source ${PATHRT}/bl_date.conf -BL_DATE=20230815 +source ${PATHRT}/bl_date.conf +#BL_DATE=20230815 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} @@ -73,7 +73,7 @@ atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configur compute_petbounds_and_tasks -atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure +atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure # diag table if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then @@ -85,7 +85,7 @@ if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then fi # Field Dictionary -cp ${PATHRT}/parm/fd_nems.yaml fd_nems.yaml +cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml # Set up the run directory source ./fv3_run From 19802292ec9f801ea5c4bed2866ab371b2af00bc Mon Sep 17 00:00:00 2001 From: rhaesung Date: Wed, 15 Nov 2023 15:42:50 +0000 Subject: [PATCH 36/65] revert to the default offline-noahmp/era5 setting --- submit_cycle.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 5bb11900..69972159 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -1,10 +1,10 @@ #!/bin/bash #SBATCH --job-name=ufs_land_da -#SBATCH --account=epic +#SBATCH --account=da-cpu #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --tasks-per-node=6 -#SBATCH --cpus-per-task=5 +#SBATCH --cpus-per-task=1 #SBATCH -t 00:06:00 #SBATCH -o log_noahmp.%j.log #SBATCH -e err_noahmp.%j.err From 07fb73afb3bf514584c6906401e7b8349031dcd2 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:56:01 -0500 Subject: [PATCH 37/65] Update Jenkinsfile jenkins workspace --- test/ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 4e15b74b..992ef787 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { } environment { ACCNR = 'nems' - NODE_PATH = '/scratch1/NCEPDEV/stmp2/role.epic/' + NODE_PATH = '/scratch2/NAGAPE/epic/role.epic/' } steps { @@ -69,7 +69,7 @@ pipeline { checkout scm sh ''' git submodule update --init --recursive - ln -fs /scratch2/NAGAPE/epic/UFS_Land-DA/inputs /scratch1/NCEPDEV/stmp2/role.epic/jenkins/workspace/ + ln -fs /scratch2/NAGAPE/epic/UFS_Land-DA/inputs /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/ module use modulefiles && module load landda_hera.intel mkdir build cp test/hera_ctest.sh build/ From bd4298a9caa3211886c4e55af3ba771591e8a6d2 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Wed, 15 Nov 2023 21:43:16 +0000 Subject: [PATCH 38/65] remove tile2tile submodule --- .gitmodules | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index e739a926..b43d0a53 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,10 +6,6 @@ path = DA_update url = https://github.com/ufs-community/land-DA.git branch = feature/porting -[submodule "tile2tile"] - path = tile2tile - url = https://github.com/rhaesung/land-tile2tile.git - branch = feature/porting [submodule "vector2tile"] path = vector2tile url = https://github.com/NOAA-PSL/land-vector2tile.git From 605b71a45e4893344a3e48a0087d9ae329a0778f Mon Sep 17 00:00:00 2001 From: rhaesung Date: Fri, 17 Nov 2023 14:19:09 +0000 Subject: [PATCH 39/65] update the RT path on Orion --- submit_cycle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 69972159..8734d6a9 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -310,7 +310,7 @@ while [ $date_count -lt $cycles_per_job ]; do # Set inputdata location for each machines echo "MACHINE_ID: $MACHINE_ID" if [[ $MACHINE_ID = orion ]]; then - DISKNM=/work/noaa/nems/emc.nemspara/RT + DISKNM=/work/noaa/epic/UFS-WM_RT elif [[ $MACHINE_ID = hera ]]; then DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT else From a2a10238596f563dfcec5f6695344839edcc7a1a Mon Sep 17 00:00:00 2001 From: rhaesung Date: Sat, 18 Nov 2023 02:29:05 +0000 Subject: [PATCH 40/65] update the GHCN path --- DA_update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DA_update b/DA_update index 31fff859..a81c8fd5 160000 --- a/DA_update +++ b/DA_update @@ -1 +1 @@ -Subproject commit 31fff859b0b1d2ce8128781b8abffa8d27e916c5 +Subproject commit a81c8fd5ec4a6c571f76ce3426f2b0777a6d2349 From a24962a62a279217fac28bc26446e912b78286d9 Mon Sep 17 00:00:00 2001 From: JONG KIM Date: Mon, 20 Nov 2023 11:58:57 -0500 Subject: [PATCH 41/65] Update landda_orion.intel.lua --- modulefiles/landda_orion.intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/landda_orion.intel.lua b/modulefiles/landda_orion.intel.lua index f4785001..bb515c04 100644 --- a/modulefiles/landda_orion.intel.lua +++ b/modulefiles/landda_orion.intel.lua @@ -2,7 +2,7 @@ help([[ loads UFS Model prerequisites for Orion/Intel ]]) -prepend_path("MODULEPATH", "/work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles") stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" From 599258fc71ad02b072fa5090b725297abd56fb7d Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 11:53:45 -0600 Subject: [PATCH 42/65] change the RT path of Orion. --- test/run_ufs_datm_lnd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_ufs_datm_lnd.sh b/test/run_ufs_datm_lnd.sh index 091d59cf..b8f7db94 100755 --- a/test/run_ufs_datm_lnd.sh +++ b/test/run_ufs_datm_lnd.sh @@ -24,7 +24,7 @@ source ${PATHRT}/atparse.bash # Set inputdata location for each machines echo "MACHINE_ID: $MACHINE_ID" if [[ $MACHINE_ID = orion ]]; then - DISKNM=/work/noaa/nems/emc.nemspara/RT + DISKNM=/work/noaa/epic/UFS-WM_RT elif [[ $MACHINE_ID = hera ]]; then DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT else From cc7557006aaa9cd9d716fcb45fae81ad6bd74619 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:57:33 -0500 Subject: [PATCH 43/65] Create orion_ctest.sh --- test/orion_ctest.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/orion_ctest.sh diff --git a/test/orion_ctest.sh b/test/orion_ctest.sh new file mode 100644 index 00000000..5484da0f --- /dev/null +++ b/test/orion_ctest.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH -o out.ctest +#SBATCH --account=nems +set -eux + +module use ../modulefiles && module landda_orion.intel.lua + +ctest + +wait + +echo "ctest is done" From bc598fd246f46b97e901067ae82be1ece95e2602 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:00:28 -0500 Subject: [PATCH 44/65] Update Jenkinsfile to add orion --- test/ci/Jenkinsfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 992ef787..e7d1ca3d 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -96,7 +96,21 @@ pipeline { cleanWs() checkout scm sh ''' - + git submodule update --init --recursive + ln -fs /work/noaa/epic/UFS_Land-DA/inputs /work/noaa/epic/role-epic/jenkins/workspace/ + module use modulefiles && module load landda_orion.intel.lua + mkdir build + cp test/orion_ctest.sh build/ + cp test/check_ctest.sh build/ + cp test/run_ctest.sh build/ + cd test/ + sed -i 's|export MACHINE_ID=${MACHINE_ID:-hera}|export MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh + cd .. + cd build/ + module use ../modulefiles && module load landda_orion.intel.lua + ecbuild ../ && make -j4 + echo $(pwd) + ./run_ctest.sh ''' } } From 4f659c8aa39629e44e00bc6917ffa470b73398b2 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:12:57 -0500 Subject: [PATCH 45/65] Rename run_ctest.sh to run_hera_ctest.sh --- test/{run_ctest.sh => run_hera_ctest.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{run_ctest.sh => run_hera_ctest.sh} (100%) diff --git a/test/run_ctest.sh b/test/run_hera_ctest.sh similarity index 100% rename from test/run_ctest.sh rename to test/run_hera_ctest.sh From 4546ff364d0ae443e854c99755a8a0391b4c20a5 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:14:11 -0500 Subject: [PATCH 46/65] Create run_orion_ctest.sh --- test/testinput/run_orion_ctest.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/testinput/run_orion_ctest.sh diff --git a/test/testinput/run_orion_ctest.sh b/test/testinput/run_orion_ctest.sh new file mode 100644 index 00000000..71319cfc --- /dev/null +++ b/test/testinput/run_orion_ctest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux + +JOB_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') + +CHECK_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) + +sleep 3m + +if [ -f out.ctest ]; then + cat out.ctest +else + echo "ctest run fails to run." +fi From 3faea1485d7e8f364a1faf7794755691ed44fd2a Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:17:26 -0500 Subject: [PATCH 47/65] Delete test/testinput/run_orion_ctest.sh --- test/testinput/run_orion_ctest.sh | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 test/testinput/run_orion_ctest.sh diff --git a/test/testinput/run_orion_ctest.sh b/test/testinput/run_orion_ctest.sh deleted file mode 100644 index 71319cfc..00000000 --- a/test/testinput/run_orion_ctest.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -eux - -JOB_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') - -CHECK_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) - -sleep 3m - -if [ -f out.ctest ]; then - cat out.ctest -else - echo "ctest run fails to run." -fi From 7a146e925336194fc1286dc4006945d6b204ddb8 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:18:04 -0500 Subject: [PATCH 48/65] Create run_orion_ctest.sh --- test/run_orion_ctest.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/run_orion_ctest.sh diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh new file mode 100644 index 00000000..71319cfc --- /dev/null +++ b/test/run_orion_ctest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux + +JOB_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') + +CHECK_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) + +sleep 3m + +if [ -f out.ctest ]; then + cat out.ctest +else + echo "ctest run fails to run." +fi From 5d177e4b1f329ceca04e13df7599783cdd075266 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:20:23 -0500 Subject: [PATCH 49/65] Update Jenkinsfile add orion scripts --- test/ci/Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index e7d1ca3d..2aebad61 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -79,7 +79,7 @@ pipeline { module use ../modulefiles && module load landda_hera.intel ecbuild ../ && make -j4 echo $(pwd) - ./run_ctest.sh + ./run_hera_ctest.sh ''' } } @@ -103,14 +103,14 @@ pipeline { cp test/orion_ctest.sh build/ cp test/check_ctest.sh build/ cp test/run_ctest.sh build/ - cd test/ - sed -i 's|export MACHINE_ID=${MACHINE_ID:-hera}|export MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh - cd .. + cd test/ + sed -i 's|export MACHINE_ID=${MACHINE_ID:-hera}|export MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh + cd .. cd build/ module use ../modulefiles && module load landda_orion.intel.lua ecbuild ../ && make -j4 echo $(pwd) - ./run_ctest.sh + ./run_orion_ctest.sh ''' } } From 9045da94636d3082b9d48cf8a5fe370cd4fc1e74 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:25:28 -0500 Subject: [PATCH 50/65] Update Jenkinsfile --- test/ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 2aebad61..29803335 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -74,7 +74,7 @@ pipeline { mkdir build cp test/hera_ctest.sh build/ cp test/check_ctest.sh build/ - cp test/run_ctest.sh build/ + cp test/run_hera_ctest.sh build/ cd build/ module use ../modulefiles && module load landda_hera.intel ecbuild ../ && make -j4 @@ -102,7 +102,7 @@ pipeline { mkdir build cp test/orion_ctest.sh build/ cp test/check_ctest.sh build/ - cp test/run_ctest.sh build/ + cp test/run_orion_ctest.sh build/ cd test/ sed -i 's|export MACHINE_ID=${MACHINE_ID:-hera}|export MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh cd .. From 59d0c04cea02454780a610aaa22e007a156e78f8 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:43:31 -0500 Subject: [PATCH 51/65] Update Jenkinsfile --- test/ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 29803335..89a7ed9d 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -110,6 +110,7 @@ pipeline { module use ../modulefiles && module load landda_orion.intel.lua ecbuild ../ && make -j4 echo $(pwd) + groups ./run_orion_ctest.sh ''' } From 5d70a7f33fae00f57882f8422a6715dfa756f21a Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 20:46:39 +0000 Subject: [PATCH 52/65] add mchine_id option in setting --- settings_DA_cycle_gswp3 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3 index 84620acd..d77cb54d 100644 --- a/settings_DA_cycle_gswp3 +++ b/settings_DA_cycle_gswp3 @@ -4,6 +4,9 @@ export exp_name=DA_GSWP3_test #export BASELINE=hera.internal +#Machine options: orion, hera +export MACHINE_ID=hera + # experiment period STARTDATE=2000010300 ENDDATE=2000010400 From fe3d9b8f7de427f5d19d780fca37347e96ebb788 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 20:47:17 +0000 Subject: [PATCH 53/65] clean up the machine_ID --- test/run_ufs_datm_lnd.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/run_ufs_datm_lnd.sh b/test/run_ufs_datm_lnd.sh index b8f7db94..5234ff35 100755 --- a/test/run_ufs_datm_lnd.sh +++ b/test/run_ufs_datm_lnd.sh @@ -10,7 +10,6 @@ echo ${project_binary_dir} echo ${project_source_dir} # -export MACHINE_ID=${MACHINE_ID:-hera} TEST_NAME=datm_cdeps_lnd_gswp3 PATHRT=${project_source_dir}/ufs-weather-model/tests RT_COMPILER=${RT_COMPILER:-intel} From 672b1dac7a33d33f8e38cafcc992ffd4c3eb6b36 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 20:48:21 +0000 Subject: [PATCH 54/65] clean up --- submit_cycle.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index 8734d6a9..af1fc1aa 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -293,7 +293,6 @@ while [ $date_count -lt $cycles_per_job ]; do echo '************************************************' echo 'running the forecast model' - export MACHINE_ID=${MACHINE_ID:-hera} TEST_NAME=datm_cdeps_lnd_gswp3 TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst PATHRT=${CYCLEDIR}/ufs-weather-model/tests From e4b8537df0cd18c736b33cfa2a60da5bea913626 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:06:25 -0500 Subject: [PATCH 55/65] Update run_orion_ctest.sh fix orion account --- test/run_orion_ctest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh index 71319cfc..dfc12290 100644 --- a/test/run_orion_ctest.sh +++ b/test/run_orion_ctest.sh @@ -1,9 +1,9 @@ #!/bin/bash set -eux -JOB_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') +JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') -CHECK_ID=$(sbatch --job-name=ctest --account=nems --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) +CHECK_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) sleep 3m From b3908b62c0ddf52c482cde3d2e61d446103bfbd2 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 21:23:40 +0000 Subject: [PATCH 56/65] add datm_cdeps_lnd_gswp3_rst file in the cycle directory --- datm_cdeps_lnd_gswp3_rst | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 datm_cdeps_lnd_gswp3_rst diff --git a/datm_cdeps_lnd_gswp3_rst b/datm_cdeps_lnd_gswp3_rst new file mode 100644 index 00000000..e001ad37 --- /dev/null +++ b/datm_cdeps_lnd_gswp3_rst @@ -0,0 +1,62 @@ +# +# DATM_CDEPS_NOAHMP_GSWP test +# + +export TEST_DESCR="DATM_CDEPS_NOAHMP_GSWP3_RST - control restart" + +export CNTL_DIR="datm_cdeps_lnd_gswp3" + +export LIST_FILES="ufs.cpld.lnd.out.2000-01-02-00000.tile1.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile2.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile3.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile4.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile5.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile6.nc" + +export_datm_cdeps + +export SYEAR=2000 +export SMONTH=01 +export SDAY=03 +export SHOUR=00 +export FHMAX=24 +export FHROT=0 + +export DATM_IN_CONFIGURE=datm_in +export MESH_ATM="INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc" +export atm_datamode="CLMNCEP" +export ATM_NX_GLB=144 +export ATM_NY_GLB=96 + +export DATM_STREAM_CONFIGURE=datm.streams.multi.IN +export MESH_ATM_DATA="INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc" +export MESH_ATM_TOPO="INPUT_DATM/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc" +export SDATE=1999 +export EDATE=2000 +export DATA_ATM01="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc\"" +export DATA_ATM02="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc\"" +export DATA_ATM03="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc\"" +export DATA_ATM04="\"INPUT_DATM/topodata_0.9x1.25_USGS_070110_stream_c151201.nc\"" + +export UFS_CONFIGURE="ufs.configure.atm_lnd.IN" +export med_model="cmeps" +export atm_model="datm" +export lnd_model="noahmp" +export CPLMODE="ufs.nfrac.aoflux" +export RUNTYPE="continue" +export READRESTART=.false. +export ATM_compute_tasks=144 +export OCN_tasks=0 +export ICE_tasks=0 +export LND_tasks=144 +export lnd_ic_type='custom' +export CALC_SNET=.true. +export layout_x=3 +export layout_y=8 +export mosaic_file="INPUT/C96_mosaic.nc" +export coupling_interval_sec=3600 +export TOPOEDITS="" +export WARM_START=.true. + +export RESTART_FILE_SUFFIX_SECS="${SYEAR}-${SMONTH}-${SDAY}-$(printf "%05d" $(( (${FHROT}+ ${SHOUR})*3600 )))" +export FV3_RUN="lnd_datm_cdeps_gswp.IN noahmp_run.IN" From d1ac9078038b1f7e40910d3d5a2e23416db4e515 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 21:24:38 +0000 Subject: [PATCH 57/65] clean up --- submit_cycle.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/submit_cycle.sh b/submit_cycle.sh index af1fc1aa..a1157a46 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -299,7 +299,7 @@ while [ $date_count -lt $cycles_per_job ]; do RT_COMPILER=${RT_COMPILER:-intel} ATOL="1e-7" - cp ${LANDDA_INPUTS}/restarts/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST + cp $CYCLEDIR/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST source ${PATHRT}/detect_machine.sh source ${PATHRT}/rt_utils.sh source ${PATHRT}/default_vars.sh @@ -317,7 +317,6 @@ while [ $date_count -lt $cycles_per_job ]; do fi source ${PATHRT}/bl_date.conf - #BL_DATE=20230815 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} From ff665b42b36112745f8ca6249d8801ff805bf7e9 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Tue, 21 Nov 2023 22:04:28 +0000 Subject: [PATCH 58/65] adding back the machine_ID for ctest --- test/run_ufs_datm_lnd.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/run_ufs_datm_lnd.sh b/test/run_ufs_datm_lnd.sh index 5234ff35..151fe650 100755 --- a/test/run_ufs_datm_lnd.sh +++ b/test/run_ufs_datm_lnd.sh @@ -10,6 +10,7 @@ echo ${project_binary_dir} echo ${project_source_dir} # +MACHINE_ID=${MACHINE_ID:-hera} TEST_NAME=datm_cdeps_lnd_gswp3 PATHRT=${project_source_dir}/ufs-weather-model/tests RT_COMPILER=${RT_COMPILER:-intel} From 37233533771d44c5099aa508ce09e8c6fa2b7585 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:29:56 -0500 Subject: [PATCH 59/65] Update Jenkinsfile --- test/ci/Jenkinsfile | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 89a7ed9d..07e7e36a 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -96,22 +96,7 @@ pipeline { cleanWs() checkout scm sh ''' - git submodule update --init --recursive - ln -fs /work/noaa/epic/UFS_Land-DA/inputs /work/noaa/epic/role-epic/jenkins/workspace/ - module use modulefiles && module load landda_orion.intel.lua - mkdir build - cp test/orion_ctest.sh build/ - cp test/check_ctest.sh build/ - cp test/run_orion_ctest.sh build/ - cd test/ - sed -i 's|export MACHINE_ID=${MACHINE_ID:-hera}|export MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh - cd .. - cd build/ - module use ../modulefiles && module load landda_orion.intel.lua - ecbuild ../ && make -j4 - echo $(pwd) - groups - ./run_orion_ctest.sh + ''' } } From 3c74eecb20523a6179e3ddd4ad09ad2b7a40e277 Mon Sep 17 00:00:00 2001 From: zach1221 Date: Tue, 21 Nov 2023 17:21:42 -0600 Subject: [PATCH 60/65] orion_ctest.sh permissions --- test/orion_ctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 test/orion_ctest.sh diff --git a/test/orion_ctest.sh b/test/orion_ctest.sh old mode 100644 new mode 100755 index 5484da0f..637de305 --- a/test/orion_ctest.sh +++ b/test/orion_ctest.sh @@ -3,7 +3,7 @@ #SBATCH --account=nems set -eux -module use ../modulefiles && module landda_orion.intel.lua + module use modulefiles && module load landda_orion.intel.lua ctest From f16ac050906a752e2be1cb88dfd6c48dddd07703 Mon Sep 17 00:00:00 2001 From: zach1221 Date: Tue, 21 Nov 2023 17:22:59 -0600 Subject: [PATCH 61/65] run_orion_ctest.sh permissions --- test/run_orion_ctest.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) mode change 100644 => 100755 test/run_orion_ctest.sh diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh old mode 100644 new mode 100755 index dfc12290..ea0ac4c8 --- a/test/run_orion_ctest.sh +++ b/test/run_orion_ctest.sh @@ -1,5 +1,4 @@ -#!/bin/bash -set -eux +et -eux JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') From 0bd3c33ba051add3f898f6f22c1020dc60ef0a84 Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 18:24:50 -0500 Subject: [PATCH 62/65] Update run_orion_ctest.sh --- test/run_orion_ctest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh index ea0ac4c8..1c4be1f6 100755 --- a/test/run_orion_ctest.sh +++ b/test/run_orion_ctest.sh @@ -1,3 +1,4 @@ +#!/bin/bash et -eux JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') From 6411436a1043aa5639e75c3f2a6d79c1fd75208b Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 19:01:21 -0500 Subject: [PATCH 63/65] Update run_orion_ctest.sh --- test/run_orion_ctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh index 1c4be1f6..dfc12290 100755 --- a/test/run_orion_ctest.sh +++ b/test/run_orion_ctest.sh @@ -1,5 +1,5 @@ #!/bin/bash -et -eux +set -eux JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') From 59f3064c46d8d05b8e77da97111c4d851258c1bc Mon Sep 17 00:00:00 2001 From: zach1221 <99902696+zach1221@users.noreply.github.com> Date: Tue, 21 Nov 2023 20:09:18 -0500 Subject: [PATCH 64/65] Update Jenkinsfile --- test/ci/Jenkinsfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index 07e7e36a..55dac041 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -96,7 +96,21 @@ pipeline { cleanWs() checkout scm sh ''' - + git submodule update --init --recursive + ln -fs /work/noaa/epic/UFS_Land-DA/inputs /work/noaa/epic/role-epic/jenkins/workspace/ + module use modulefiles && module load landda_orion.intel.lua + mkdir build + cp test/orion_ctest.sh build/ + cp test/check_ctest.sh build/ + cp test/run_orion_ctest.sh build/ + cd test/ + sed -i 's|MACHINE_ID=${MACHINE_ID:-hera}|MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh + cd .. + cd build/ + module use ../modulefiles && module load landda_orion.intel.lua + ecbuild ../ && make -j4 + echo $(pwd) + ctest ''' } } From ea623d96727cc5448981cbd1c46d5c64e089de46 Mon Sep 17 00:00:00 2001 From: rhaesung Date: Wed, 22 Nov 2023 20:43:22 +0000 Subject: [PATCH 65/65] remove README for tile2tile --- tile2tile/README.md | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 tile2tile/README.md diff --git a/tile2tile/README.md b/tile2tile/README.md deleted file mode 100644 index 69c46d60..00000000 --- a/tile2tile/README.md +++ /dev/null @@ -1,30 +0,0 @@ -Code to map between vector format used by the Noah-MP offline driver, and the tile format used by the UFS atmospheric model. Currently used to prepare input tile files for JEDI. These files include only those fields required by JEDI, rather than the full restart. Can also be used to map stochastic_physics output to the tile or vector. Stochastic physics output files depend on the layout, with 1x1 layout giving one file per tile. - -Mike Barlage, Clara Draper. Dec 2021. - -To compile on hera: - ->configure - - choose hera - - load the modules indicated - ->make - -To run: - ->vector2tile_converter.exe namelist.vector2tile - -the namelist defines the conversion direction and the paths of the files - -Details: - -the vector2tile pathway assumes that the vector file exists in the vector_restart_path directory and overwrites/creates tile files in the output_path - -the tile2vector pathway is a little tricky, it assumes the tile files exist in tile_restart_path and overwrites only the snow variables in the vector file in the output_path. If the vector file does not exist in output_path the process will fail. - -the overall assumption here is that we will have a full model vector restart file, then convert the vector to tiles for only snow variables, then convert the updated snow variables back to the full vector restart file - -For the lndp2vector or lndp2tile option, a new file will be created with the pertbations and lat lon only. -