Porting UFS-WM Noah-MP GSWP3 baseline to land DA workflow (#32)

* Update Jenkinsfile adding sonarqube * add jedi2ufs conversion feature * add ufs2jedi and jedi2ufs tile conversion * remove unnecessary templates * add new tile2tile templates * add the fake_observations * Read in customized UFS datm_cdeps_lnd_gswp3_rst RT script and add write option of the UFS-land restart file as background file * update the GHCN path * Update landda_orion.intel.lua * Create run_orion_ctest.sh * Update Jenkinsfile add orion scripts * add datm_cdeps_lnd_gswp3_rst file in the cycle directory --------- Co-authored-by: zach1221 <[email protected]> Co-authored-by: JONG KIM <[email protected]> Co-authored-by: zach1221 <[email protected]>
ufs-community · Nov 24, 2023 · 5ca8700 · 5ca8700
1 parent 8c0f8de
commit 5ca8700
Show file tree

Hide file tree

Showing 54 changed files with 4,011 additions and 88 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -5,7 +5,7 @@
 [submodule "DA_update"]
 	path = DA_update
         url = https://github.com/ufs-community/land-DA.git
-        branch = develop
+        branch = feature/porting
 [submodule "vector2tile"]
 	path = vector2tile
 	url = https://github.com/NOAA-PSL/land-vector2tile.git

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -55,6 +55,7 @@ ExternalProject_Add(ufs-weather-model
   )
 
 add_subdirectory( vector2tile )
+add_subdirectory( tile2tile )
 add_subdirectory( ufs-land-driver-emc-dev )
 add_subdirectory( DA_update/add_jedi_incr )
 

diff --git a/DA_update b/DA_update
diff --git a/datm_cdeps_lnd_gswp3_rst b/datm_cdeps_lnd_gswp3_rst
@@ -0,0 +1,62 @@
+#
+#  DATM_CDEPS_NOAHMP_GSWP test
+#
+
+export TEST_DESCR="DATM_CDEPS_NOAHMP_GSWP3_RST - control restart"
+
+export CNTL_DIR="datm_cdeps_lnd_gswp3"
+
+export LIST_FILES="ufs.cpld.lnd.out.2000-01-02-00000.tile1.nc \
+                   ufs.cpld.lnd.out.2000-01-02-00000.tile2.nc \
+                   ufs.cpld.lnd.out.2000-01-02-00000.tile3.nc \
+                   ufs.cpld.lnd.out.2000-01-02-00000.tile4.nc \
+                   ufs.cpld.lnd.out.2000-01-02-00000.tile5.nc \
+                   ufs.cpld.lnd.out.2000-01-02-00000.tile6.nc"
+
+export_datm_cdeps
+
+export SYEAR=2000
+export SMONTH=01
+export SDAY=03
+export SHOUR=00
+export FHMAX=24
+export FHROT=0
+
+export DATM_IN_CONFIGURE=datm_in
+export MESH_ATM="INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc"
+export atm_datamode="CLMNCEP"
+export ATM_NX_GLB=144
+export ATM_NY_GLB=96
+
+export DATM_STREAM_CONFIGURE=datm.streams.multi.IN
+export MESH_ATM_DATA="INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc"
+export MESH_ATM_TOPO="INPUT_DATM/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc"
+export SDATE=1999
+export EDATE=2000
+export DATA_ATM01="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc\""
+export DATA_ATM02="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc\""
+export DATA_ATM03="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc\""
+export DATA_ATM04="\"INPUT_DATM/topodata_0.9x1.25_USGS_070110_stream_c151201.nc\""
+
+export UFS_CONFIGURE="ufs.configure.atm_lnd.IN"
+export med_model="cmeps"
+export atm_model="datm"
+export lnd_model="noahmp"
+export CPLMODE="ufs.nfrac.aoflux"
+export RUNTYPE="continue"
+export READRESTART=.false.
+export ATM_compute_tasks=144
+export OCN_tasks=0
+export ICE_tasks=0
+export LND_tasks=144
+export lnd_ic_type='custom'
+export CALC_SNET=.true.
+export layout_x=3
+export layout_y=8
+export mosaic_file="INPUT/C96_mosaic.nc"
+export coupling_interval_sec=3600
+export TOPOEDITS=""
+export WARM_START=.true.
+
+export RESTART_FILE_SUFFIX_SECS="${SYEAR}-${SMONTH}-${SDAY}-$(printf "%05d" $(( (${FHROT}+ ${SHOUR})*3600 )))"
+export FV3_RUN="lnd_datm_cdeps_gswp.IN noahmp_run.IN"
diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh
@@ -20,7 +20,6 @@ source $config_file
 
 export KEEPWORKDIR="YES"
 
-
 ############################
 # ensure necessary envars are set
 envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "CYCLEDIR" \
@@ -77,9 +76,14 @@ fi
 
 if [[ -e ${BUILDDIR}/bin/vector2tile_converter.exe ]]; then #prefer cmake-built executables
   export vec2tileexec=${BUILDDIR}/bin/vector2tile_converter.exe
-else 
+else
   export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe
 fi
+if [[ -e ${BUILDDIR}/bin/tile2tile_converter.exe ]]; then #prefer cmake-built executables
+  export tile2tileexec=${BUILDDIR}/bin/tile2tile_converter.exe
+else 
+  export tile2tileexec=${CYCLEDIR}/tile2tile/tile2tile_converter.exe
+fi
 if [[ -e ${BUILDDIR}/bin/ufsLand.exe ]]; then
   export LSMexec=${BUILDDIR}/bin/ufsLand.exe
 else
@@ -154,20 +158,42 @@ fi
 ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output 
 
 # copy ICS into restarts, if needed 
-rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
-rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
-# if restart not in experiment out directory, copy the restarts from the ICSDIR
-if [[ ! -e ${rst_out} ]]; then
-    echo "Looking for ICS: ${rst_in}"
-    if [[ -e ${rst_in} ]]; then
-       echo "ICS found, copying" 
-       cp ${rst_in} ${rst_out}
-    else
-       echo "ICS not found. Exiting" 
-       exit 10
+if [[ $atmos_forc == "era5" ]]; then
+    rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+    rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc
+    # if restart not in experiment out directory, copy the restarts from the ICSDIR
+    if [[ ! -e ${rst_out} ]]; then
+        echo "Looking for ICS: ${rst_in}"
+    	if [[ -e ${rst_in} ]]; then
+            echo "ICS found, copying" 
+            cp ${rst_in} ${rst_out}
+        else
+            echo "ICS not found. Exiting" 
+            exit 10
+        fi
     fi
 fi
 
+if [[ $atmos_forc == "gswp3" ]]; then
+
+   for tile in 1 2 3 4 5 6
+   do
+   rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc
+   rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${sYYYY}-${sMM}-${sDD}-00000.tile${tile}.nc
+   # if restart not in experiment out directory, copy the restarts from the ICSDIR
+   if [[ ! -e ${rst_out} ]]; then
+       echo "Looking for ICS: ${rst_in}"
+       if [[ -e ${rst_in} ]]; then
+           echo "ICS found, copying" 
+           cp ${rst_in} ${rst_out}
+       else
+           echo "ICS not found. Exiting" 
+           exit 10
+       fi
+   fi
+   done
+fi
+
 # create dates file 
 touch analdates.sh 
 cat << EOF > analdates.sh

diff --git a/modulefiles/landda_orion.intel.lua b/modulefiles/landda_orion.intel.lua
@@ -2,7 +2,7 @@ help([[
 loads UFS Model prerequisites for Orion/Intel
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"

diff --git a/settings_DA_cycle_era5 b/settings_DA_cycle_era5
@@ -1,7 +1,7 @@
 # Settings file for submit_cycle, for running the DA_GHCN_test 
 
 # experiment name
-export exp_name=DA_GHCN_test
+export exp_name=DA_ERA5_test
 #export BASELINE=hera.internal
 
 #ERA5 forcing is available for 2020-2021
@@ -12,7 +12,7 @@ ENDDATE=2019122200
 source ./release.environment
 ############################
 
-#forcing options: gdas, era5
+#forcing options: gswp3, era5
 export atmos_forc=era5
 
 # for LETKF, this is size of ensemble.

diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3
@@ -0,0 +1,53 @@
+# Settings file for submit_cycle, for running the DA_GHCN_test 
+
+# experiment name
+export exp_name=DA_GSWP3_test
+#export BASELINE=hera.internal
+
+#Machine options: orion, hera
+export MACHINE_ID=hera
+
+# experiment period 
+STARTDATE=2000010300
+ENDDATE=2000010400
+
+# Get commmon variables
+source ./release.environment
+############################
+
+#forcing options: gswp3, era5
+export atmos_forc=gswp3
+
+# for LETKF, this is size of ensemble.
+# for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 
+export ensemble_size=1
+
+# length of each forecast
+export FCSTHR=24
+
+#FV3 resolution
+export RES=96
+if [[ $BASELINE =~ 'hera.internal' ]]; then
+   export TPATH=/scratch2/NCEPDEV/land/data/fix/C96.mx100_frac/
+else
+   export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/"
+fi
+export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH
+                             # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean.
+
+# number of cycles to submit in a single job
+export cycles_per_job=1
+
+# directory with initial conditions
+# can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single
+export ICSDIR=$LANDDAROOT/inputs/forcing/${atmos_forc}/orog_files/
+
+# namelist for do_landDA.sh
+# set to "openloop" to not call do_landDA.sh
+export DA_config="settings_DA_test"
+
+# if want different DA at different times, list here. 
+export DA_config00=${DA_config} 
+export DA_config06=${DA_config} 
+export DA_config12=${DA_config} 
+export DA_config18=${DA_config}