Skip to content

Commit

Permalink
Refactor fv3atm history & restart to reduce redundant code. Add rrfs-…
Browse files Browse the repository at this point in the history 
…sd and clm lake to quilt restart. (ufs-community#1769)

* refactor FV3GFS restart, add clm lake and rrfs-sd to quilt restart

* rrfs sd restart in quilt (still under testing)

* explicitly write axis values (from DusanJovic-NOAA), and clean up code

* rename FV3GFS to fv3atm in code, except output metadata

* refactor a bit more, putting history into fv3atm_history_io.F90 and restart into fv3atm_restart_io.F90

* add the two qr hurr_control tests

* Even longer explanation of RUC LSM axes from @tanyasmirnova

* missing dependency for rrfs_smoke_conus13km_hrrr_warm_restart

* do not delete tests/lock directory when another rt.sh is running

* Set THRD variable in rrfs 2threads tests

* bug fix to "alt check" code: add -S -q to avoid infinite output

* 32-bit quilt restart tests

* use process id instead of $delete_lock to decide3 whether to delete the lock file

* Remove THRD= from smoke tests

* no THRD in non-smoke conus13km tests

* save restarts in rrfs_conus13km_hrrr_warm test

* disable four broken tests
  • Loading branch information
@SamuelTrahanNOAA
SamuelTrahanNOAA authored Jun 20, 2023
1 parent 251ec56 commit 3f82945
Show file tree
Hide file tree
Showing 24 changed files with 11,616 additions and 8,611 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ endif()

if(FV3)
add_dependencies(ufs fv3atm)
list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
list(APPEND _ufs_defs_private FRONT_FV3=fv3atm_cap_mod)
list(APPEND _ufs_libs_public fv3atm)
endif()

Expand Down
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 16 files
+7 −3 CMakeLists.txt
+15 −14 atmos_model.F90
+9 −9 fv3_cap.F90
+0 −4,325 io/FV3GFS_io.F90
+0 −920 io/FV3GFS_restart_io.F90
+0 −432 io/clm_lake_io.F90
+521 −0 io/fv3atm_clm_lake_io.F90
+518 −0 io/fv3atm_common_io.F90
+1,184 −0 io/fv3atm_history_io.F90
+333 −0 io/fv3atm_oro_io.F90
+1,282 −0 io/fv3atm_restart_io.F90
+607 −0 io/fv3atm_rrfs_sd_io.F90
+1,625 −0 io/fv3atm_sfc_io.F90
+2 −2 io/module_wrt_grid_comp.F90
+2 −2 module_fcst_grid_comp.F90
+1 −1 moving_nest/fv_moving_nest_physics.F90
2 changes: 1 addition & 1 deletion tests/bl_date.conf
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export BL_DATE=20230608
export BL_DATE=20230616

3,846 changes: 2,113 additions & 1,733 deletions tests/logs/RegressionTests_acorn.log
View file

Large diffs are not rendered by default.

3,140 changes: 1,755 additions & 1,385 deletions tests/logs/RegressionTests_cheyenne.log
View file

Large diffs are not rendered by default.

2,005 changes: 1,095 additions & 910 deletions tests/logs/RegressionTests_gaea.log
View file

Large diffs are not rendered by default.

3,106 changes: 1,738 additions & 1,368 deletions tests/logs/RegressionTests_hera.log
View file

Large diffs are not rendered by default.

2,245 changes: 1,340 additions & 905 deletions tests/logs/RegressionTests_jet.log
View file

Large diffs are not rendered by default.

2,262 changes: 1,198 additions & 1,064 deletions tests/logs/RegressionTests_orion.log
View file

Large diffs are not rendered by default.

2,631 changes: 1,396 additions & 1,235 deletions tests/logs/RegressionTests_wcoss2.log
View file

Large diffs are not rendered by default.

18 changes: 15 additions & 3 deletions tests/rt.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,21 +122,25 @@ RUN | rap_sfcdiff |
RUN | rap_sfcdiff_decomp | | |
RUN | rap_sfcdiff_restart | | | rap_sfcdiff
RUN | hrrr_control | | baseline |
RUN | hrrr_control_qr | | baseline |
RUN | hrrr_control_decomp | | |
RUN | hrrr_control_2threads | | |
RUN | hrrr_control_restart | | | hrrr_control
RUN | hrrr_control_restart_qr | | | hrrr_control_qr
RUN | rrfs_v1beta | | baseline |
RUN | rrfs_v1nssl | | baseline |
RUN | rrfs_v1nssl_nohailnoccn | | baseline |
RUN | rrfs_smoke_conus13km_hrrr_warm | | baseline |
#RUN | rrfs_smoke_conus13km_hrrr_warm_qr | | |
RUN | rrfs_smoke_conus13km_hrrr_warm_2threads | | |
RUN | rrfs_conus13km_hrrr_warm | | baseline |
RUN | rrfs_smoke_conus13km_radar_tten_warm | | baseline |
# These two should run and not crash, but they do not match their baselines:
#RUN | rrfs_smoke_conus13km_hrrr_warm_decomp | | |
#RUN | rrfs_smoke_conus13km_hrrr_warm_restart | | | rrfs_smoke_conus13km_hrrr_warm
# Just to make sure restart doesn't crash again:
RUN | rrfs_conus13km_hrrr_warm_restart_mismatch | | baseline | rrfs_conus13km_hrrr_warm
RUN | rrfs_smoke_conus13km_hrrr_warm_restart_mismatch | | baseline | rrfs_smoke_conus13km_hrrr_warm
#RUN | rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch | | | rrfs_smoke_conus13km_hrrr_warm_qr

COMPILE | 14 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_csawmg,FV3_GFS_v16_ugwpv1,FV3_GFS_v16_ras,FV3_GFS_v16_noahmp | | fv3 |
RUN | control_csawmg | - gaea.intel | baseline |
Expand Down Expand Up @@ -191,11 +195,13 @@ COMPILE | 19 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR,FV3_HRRR_flake -
RUN | regional_spp_sppt_shum_skeb_dyn32_phy32 | | baseline |
RUN | rap_control_dyn32_phy32 | | baseline |
RUN | hrrr_control_dyn32_phy32 | | baseline |
RUN | hrrr_control_qr_dyn32_phy32 | | baseline |
RUN | rap_2threads_dyn32_phy32 | | |
RUN | hrrr_control_2threads_dyn32_phy32 | | |
RUN | hrrr_control_decomp_dyn32_phy32 | | |
RUN | rap_restart_dyn32_phy32 | | | rap_control_dyn32_phy32
RUN | hrrr_control_restart_dyn32_phy32 | | | hrrr_control_dyn32_phy32
RUN | hrrr_control_restart_qr_dyn32_phy32 | | | hrrr_control_qr_dyn32_phy32

COMPILE | 20 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON | | fv3 |
RUN | rap_control_dyn64_phy32 | | baseline |
Expand Down Expand Up @@ -314,19 +320,23 @@ RUN | rap_sfcdiff | + hera cheyenne
RUN | rap_sfcdiff_decomp | + hera cheyenne | |
RUN | rap_sfcdiff_restart | + hera cheyenne | | rap_sfcdiff
RUN | hrrr_control | + hera cheyenne | baseline |
RUN | hrrr_control_qr | + hera cheyenne | baseline |
RUN | hrrr_control_2threads | + hera cheyenne | |
RUN | hrrr_control_decomp | + hera cheyenne | |
RUN | hrrr_control_restart | + hera cheyenne | | hrrr_control
RUN | hrrr_control_restart_qr | + hera cheyenne | | hrrr_control_qr
RUN | rrfs_v1beta | + hera cheyenne | baseline |
RUN | rrfs_smoke_conus13km_hrrr_warm | + hera cheyenne | baseline |
#RUN | rrfs_smoke_conus13km_hrrr_warm_qr | + hera cheyenne | |
RUN | rrfs_smoke_conus13km_hrrr_warm_2threads | + hera cheyenne | |
RUN | rrfs_conus13km_hrrr_warm | + hera cheyenne | baseline |
RUN | rrfs_smoke_conus13km_radar_tten_warm | + hera cheyenne | baseline |
RUN | rrfs_conus13km_hrrr_warm | + hera cheyenne | baseline |
# These two will run and not crash, but they do not match their baselines:
#RUN | rrfs_smoke_conus13km_hrrr_warm_decomp | + hera cheyenne | |
#RUN | rrfs_smoke_conus13km_hrrr_warm_restart | + hera cheyenne | | rrfs_smoke_conus13km_hrrr_warm
# Just to make sure restart doesn't crash again:
RUN | rrfs_conus13km_hrrr_warm_restart_mismatch | + hera cheyenne | baseline | rrfs_conus13km_hrrr_warm
RUN | rrfs_smoke_conus13km_hrrr_warm_restart_mismatch | + hera cheyenne | baseline | rrfs_smoke_conus13km_hrrr_warm
#RUN | rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch | + hera cheyenne | | rrfs_smoke_conus13km_hrrr_warm_qr

### CCPP DEBUG tests ###
# Exercise compilation without specifying suites (i.e. compile all suites) in DEBUG mode (faster than in PROD mode)
Expand Down Expand Up @@ -358,11 +368,13 @@ RUN | control_wam_debug | + hera cheyenne
COMPILE | 42 | gnu | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR,FV3_HRRR_flake -D32BIT=ON -DCCPP_32BIT=ON | + hera cheyenne | fv3 |
RUN | rap_control_dyn32_phy32 | + hera cheyenne | baseline |
RUN | hrrr_control_dyn32_phy32 | + hera cheyenne | baseline |
RUN | hrrr_control_qr_dyn32_phy32 | + hera cheyenne | baseline |
RUN | rap_2threads_dyn32_phy32 | + hera cheyenne | |
RUN | hrrr_control_2threads_dyn32_phy32 | + hera cheyenne | |
RUN | hrrr_control_decomp_dyn32_phy32 | + hera cheyenne | |
RUN | rap_restart_dyn32_phy32 | + hera cheyenne | | rap_control_dyn32_phy32
RUN | hrrr_control_restart_dyn32_phy32 | + hera cheyenne | | hrrr_control_dyn32_phy32
RUN | hrrr_control_restart_qr_dyn32_phy32 | + hera cheyenne | | hrrr_control_qr_dyn32_phy32

COMPILE | 43 | gnu | -DAPP=ATM -DCCPP_32BIT=ON | + hera cheyenne | fv3 |
RUN | rap_control_dyn64_phy32 | + hera cheyenne | baseline |
Expand Down
2 changes: 1 addition & 1 deletion tests/rt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ rt_trap() {
}

cleanup() {
rm -rf ${LOCKDIR}
[[ $(awk '{print $2}' < "${LOCKDIR}/PID") == $$ ]] && rm -rf ${LOCKDIR}
[[ ${ECFLOW:-false} == true ]] && ecflow_stop
trap 0
exit
Expand Down
4 changes: 2 additions & 2 deletions tests/rt_utils.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,9 +352,9 @@ check_results() {
printf ".......ALT CHECK.."
if [[ ${MACHINE_ID} =~ orion || ${MACHINE_ID} =~ hera || ${MACHINE_ID} =~ gaea || ${MACHINE_ID} =~ jet || ${MACHINE_ID} =~ cheyenne ]] ; then
if [[ $CMP_DATAONLY == false ]]; then
nccmp -d -f -g -B --Attribute=checksum --warn=format ${RTPWD}/${CNTL_DIR}_${RT_COMPILER}/${i} ${RUNDIR}/${i} > ${i}_nccmp.log 2>&1 && d=$? || d=$?
nccmp -d -S -q -f -g -B --Attribute=checksum --warn=format ${RTPWD}/${CNTL_DIR}_${RT_COMPILER}/${i} ${RUNDIR}/${i} > ${i}_nccmp.log 2>&1 && d=$? || d=$?
else
nccmp -d -f -B --Attribute=checksum --warn=format ${RTPWD}/${CNTL_DIR}_${RT_COMPILER}/${i} ${RUNDIR}/${i} > ${i}_nccmp.log 2>&1 && d=$? || d=$?
nccmp -d -S -q -f -B --Attribute=checksum --warn=format ${RTPWD}/${CNTL_DIR}_${RT_COMPILER}/${i} ${RUNDIR}/${i} > ${i}_nccmp.log 2>&1 && d=$? || d=$?
fi
if [[ $d -ne 0 && $d -ne 1 ]]; then
echo "....ERROR" >> ${RT_LOG}
Expand Down
111 changes: 111 additions & 0 deletions tests/tests/hrrr_control_qr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
###############################################################################
#
# HRRR control test: atmosphere only at C96L127 with quilt restart
#
###############################################################################

export TEST_DESCR="Compare HRRR CLM Lake results with previous trunk version using quilting restart in atm"

export CNTL_DIR=hrrr_control

export LIST_FILES="sfcf000.nc \
sfcf009.nc \
sfcf012.nc \
atmf000.nc \
atmf009.nc \
atmf012.nc \
GFSFLX.GrbF00 \
GFSFLX.GrbF09 \
GFSFLX.GrbF12 \
GFSPRS.GrbF00 \
GFSPRS.GrbF09 \
GFSPRS.GrbF12 \
RESTART/20210322.120000.coupler.res \
RESTART/20210322.120000.fv_core.res.nc \
RESTART/20210322.120000.fv_core.res.tile1.nc \
RESTART/20210322.120000.fv_core.res.tile2.nc \
RESTART/20210322.120000.fv_core.res.tile3.nc \
RESTART/20210322.120000.fv_core.res.tile4.nc \
RESTART/20210322.120000.fv_core.res.tile5.nc \
RESTART/20210322.120000.fv_core.res.tile6.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile1.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile2.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile3.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile4.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile5.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile6.nc \
RESTART/20210322.120000.fv_tracer.res.tile1.nc \
RESTART/20210322.120000.fv_tracer.res.tile2.nc \
RESTART/20210322.120000.fv_tracer.res.tile3.nc \
RESTART/20210322.120000.fv_tracer.res.tile4.nc \
RESTART/20210322.120000.fv_tracer.res.tile5.nc \
RESTART/20210322.120000.fv_tracer.res.tile6.nc \
RESTART/20210322.120000.phy_data.tile1.nc \
RESTART/20210322.120000.phy_data.tile2.nc \
RESTART/20210322.120000.phy_data.tile3.nc \
RESTART/20210322.120000.phy_data.tile4.nc \
RESTART/20210322.120000.phy_data.tile5.nc \
RESTART/20210322.120000.phy_data.tile6.nc \
RESTART/20210322.120000.sfc_data.tile1.nc \
RESTART/20210322.120000.sfc_data.tile2.nc \
RESTART/20210322.120000.sfc_data.tile3.nc \
RESTART/20210322.120000.sfc_data.tile4.nc \
RESTART/20210322.120000.sfc_data.tile5.nc \
RESTART/20210322.120000.sfc_data.tile6.nc"

export_fv3
export NPZ=127
export NPZP=128
export DT_ATMOS=300
export SYEAR=2021
export SMONTH=03
export SDAY=22
export SHOUR=06
export RESTART_INTERVAL="6 -1"
export OUTPUT_GRID='gaussian_grid'
export NSTF_NAME='2,0,0,0,0'
export WRITE_DOPOST=.true.
export IAER=5111
export OUTPUT_FH='0 09 12'
export QUILTING_RESTART=.true.
export FRAC_GRID=.false.
export FRAC_ICE=.true.

export FV3_RUN=lake_control_run.IN
export CCPP_SUITE=FV3_HRRR
export INPUT_NML=rap.nml.IN
export FIELD_TABLE=field_table_thompson_aero_tke
export NEW_DIAGTABLE=diag_table_rap

export SFCLAY_COMPUTE_FLUX=.true.

export LKM=1
export IOPT_LAKE=2
export IMP_PHYSICS=8
export DNATS=0
export DO_SAT_ADJ=.false.
export LRADAR=.true.
export LTAEROSOL=.true.
export IALB=2
export IEMS=2
export HYBEDMF=.false.
export DO_MYNNEDMF=.true.
export DO_MYNNSFCLAY=.true.
export DO_DEEP=.false.
export SHAL_CNV=.false.
export IMFSHALCNV=-1
export IMFDEEPCNV=-1
export LHEATSTRG=.false.
export LSM=3
export LSOIL_LSM=9
export KICE=9

export GWD_OPT=3
export DO_UGWP_V0=.false.
export DO_UGWP_V0_OROG_ONLY=.false.
export DO_GSL_DRAG_LS_BL=.true.
export DO_GSL_DRAG_SS=.true.
export DO_GSL_DRAG_TOFD=.true.
export DO_UGWP_V1=.false.
export DO_UGWP_V1_OROG_ONLY=.false.

112 changes: 112 additions & 0 deletions tests/tests/hrrr_control_qr_dyn32_phy32
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
###############################################################################
#
# HRRR clm lake test: atmosphere only at C96L127
#
###############################################################################

export TEST_DESCR="Compare HRRR CLM Lake results with previous trunk version"

export CNTL_DIR=hrrr_control_qr_dyn32_phy32

export LIST_FILES="sfcf000.nc \
sfcf009.nc \
sfcf012.nc \
atmf000.nc \
atmf009.nc \
atmf012.nc \
GFSFLX.GrbF00 \
GFSFLX.GrbF09 \
GFSFLX.GrbF12 \
GFSPRS.GrbF00 \
GFSPRS.GrbF09 \
GFSPRS.GrbF12 \
RESTART/20210322.120000.coupler.res \
RESTART/20210322.120000.fv_core.res.nc \
RESTART/20210322.120000.fv_core.res.tile1.nc \
RESTART/20210322.120000.fv_core.res.tile2.nc \
RESTART/20210322.120000.fv_core.res.tile3.nc \
RESTART/20210322.120000.fv_core.res.tile4.nc \
RESTART/20210322.120000.fv_core.res.tile5.nc \
RESTART/20210322.120000.fv_core.res.tile6.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile1.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile2.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile3.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile4.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile5.nc \
RESTART/20210322.120000.fv_srf_wnd.res.tile6.nc \
RESTART/20210322.120000.fv_tracer.res.tile1.nc \
RESTART/20210322.120000.fv_tracer.res.tile2.nc \
RESTART/20210322.120000.fv_tracer.res.tile3.nc \
RESTART/20210322.120000.fv_tracer.res.tile4.nc \
RESTART/20210322.120000.fv_tracer.res.tile5.nc \
RESTART/20210322.120000.fv_tracer.res.tile6.nc \
RESTART/20210322.120000.phy_data.tile1.nc \
RESTART/20210322.120000.phy_data.tile2.nc \
RESTART/20210322.120000.phy_data.tile3.nc \
RESTART/20210322.120000.phy_data.tile4.nc \
RESTART/20210322.120000.phy_data.tile5.nc \
RESTART/20210322.120000.phy_data.tile6.nc \
RESTART/20210322.120000.sfc_data.tile1.nc \
RESTART/20210322.120000.sfc_data.tile2.nc \
RESTART/20210322.120000.sfc_data.tile3.nc \
RESTART/20210322.120000.sfc_data.tile4.nc \
RESTART/20210322.120000.sfc_data.tile5.nc \
RESTART/20210322.120000.sfc_data.tile6.nc"

export_fv3
export NPZ=127
export NPZP=128
export DT_ATMOS=300
export SYEAR=2021
export SMONTH=03
export SDAY=22
export SHOUR=06
export RESTART_INTERVAL="6 -1"
export OUTPUT_GRID='gaussian_grid'
export NSTF_NAME='2,0,0,0,0'
export WRITE_DOPOST=.true.
export IAER=5111
export OUTPUT_FH='0 09 12'
export FHMAX=12
export QUILTING_RESTART=.true.
export FRAC_GRID=.false.
export FRAC_ICE=.true.

export FV3_RUN=lake_control_run.IN
export CCPP_SUITE=FV3_HRRR
export INPUT_NML=rap.nml.IN
export FIELD_TABLE=field_table_thompson_aero_tke
export NEW_DIAGTABLE=diag_table_rap

export SFCLAY_COMPUTE_FLUX=.true.
export LHEATSTRG=.false.
export LKM=1
export IOPT_LAKE=2
export IMP_PHYSICS=8
export DNATS=0
export DO_SAT_ADJ=.false.
export LRADAR=.true.
export LTAEROSOL=.true.
export IALB=2
export IEMS=2
export HYBEDMF=.false.
export DO_MYNNEDMF=.true.
export DO_MYNNSFCLAY=.true.
export DO_DEEP=.false.
export SHAL_CNV=.false.
export IMFSHALCNV=-1
export IMFDEEPCNV=-1
export LHEATSTRG=.false.
export LSM=3
export LSOIL_LSM=9
export KICE=9

export GWD_OPT=3
export DO_UGWP_V0=.false.
export DO_UGWP_V0_OROG_ONLY=.false.
export DO_GSL_DRAG_LS_BL=.true.
export DO_GSL_DRAG_SS=.true.
export DO_GSL_DRAG_TOFD=.true.
export DO_UGWP_V1=.false.
export DO_UGWP_V1_OROG_ONLY=.false.

Loading

0 comments on commit 3f82945

Please sign in to comment.