Skip to content

Commit

Permalink
Multiple domains quilting restart (ufs-community#1722)
Browse files Browse the repository at this point in the history
* Add hafs tests that use quilting restart

* Change Orion log file permissions
  • Loading branch information
DusanJovic-NOAA authored May 19, 2023
1 parent 2cf2227 commit 52cee26
Show file tree
Hide file tree
Showing 21 changed files with 7,702 additions and 5,335 deletions.
2 changes: 1 addition & 1 deletion FV3
484 changes: 242 additions & 242 deletions tests/RegressionTests_cheyenne.gnu.log

Large diffs are not rendered by default.

1,732 changes: 937 additions & 795 deletions tests/RegressionTests_cheyenne.intel.log

Large diffs are not rendered by default.

1,877 changes: 1,106 additions & 771 deletions tests/RegressionTests_gaea.intel.log

Large diffs are not rendered by default.

486 changes: 243 additions & 243 deletions tests/RegressionTests_hera.gnu.log

Large diffs are not rendered by default.

2,175 changes: 1,214 additions & 961 deletions tests/RegressionTests_hera.intel.log

Large diffs are not rendered by default.

1,252 changes: 626 additions & 626 deletions tests/RegressionTests_jet.intel.log

Large diffs are not rendered by default.

1,975 changes: 1,155 additions & 820 deletions tests/RegressionTests_orion.intel.log
100755 → 100644

Large diffs are not rendered by default.

2,230 changes: 1,359 additions & 871 deletions tests/RegressionTests_wcoss2.intel.log

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions tests/default_vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,7 @@ export RESTART_INTERVAL=0
export FHROT=0
export coupling_interval_fast_sec=0
export QUILTING=.true.
export QUILTING_RESTART=.false.
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=6
export OUTPUT_HISTORY=.true.
Expand Down
1 change: 1 addition & 0 deletions tests/parm/model_configure_hafs.IN
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ restart_interval: @[RESTART_INTERVAL]
output_1st_tstep_rst: .false.

quilting: @[QUILTING]
quilting_restart: @[QUILTING_RESTART]
write_groups: @[WRITE_GROUP]
write_tasks_per_group: @[WRTTASK_PER_GROUP]
num_files: @[NUM_FILES]
Expand Down
4 changes: 4 additions & 0 deletions tests/rt.conf
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,15 @@ RUN | hafs_regional_atm_ocn
RUN | hafs_regional_atm_wav | | fv3 |
RUN | hafs_regional_atm_ocn_wav | | fv3 |
RUN | hafs_regional_1nest_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_regional_1nest_atm_qr | - jet.intel s4.intel cheyenne.intel | |
RUN | hafs_regional_telescopic_2nests_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_global_1nest_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_global_1nest_atm_qr | - jet.intel s4.intel cheyenne.intel | |
RUN | hafs_global_multiple_4nests_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_global_multiple_4nests_atm_qr | - jet.intel s4.intel cheyenne.intel | |
RUN | hafs_regional_specified_moving_1nest_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_regional_storm_following_1nest_atm | - jet.intel s4.intel | fv3 |
RUN | hafs_regional_storm_following_1nest_atm_qr | - jet.intel s4.intel cheyenne.intel | |
RUN | hafs_regional_storm_following_1nest_atm_ocn | - jet.intel s4.intel | fv3 |
RUN | hafs_global_storm_following_1nest_atm | - jet.intel s4.intel | fv3 |

Expand Down
2 changes: 1 addition & 1 deletion tests/rt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
fi


BL_DATE=20230515
BL_DATE=20230517

RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}

Expand Down
43 changes: 42 additions & 1 deletion tests/tests/hafs_global_1nest_atm
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,48 @@ export CNTL_DIR=hafs_global_1nest_atm
export LIST_FILES="atmf006.nc \
sfcf006.nc \
atm.nest02.f006.nc \
sfc.nest02.f006.nc"
sfc.nest02.f006.nc \
RESTART/20200825.180000.coupler.res \
RESTART/20200825.180000.fv_core.res.nc \
RESTART/20200825.180000.fv_core.res.nest02.nc \
RESTART/20200825.180000.fv_core.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_core.res.tile1.nc \
RESTART/20200825.180000.fv_core.res.tile2.nc \
RESTART/20200825.180000.fv_core.res.tile3.nc \
RESTART/20200825.180000.fv_core.res.tile4.nc \
RESTART/20200825.180000.fv_core.res.tile5.nc \
RESTART/20200825.180000.fv_core.res.tile6.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile1.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile2.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile3.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile4.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile5.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile6.nc \
RESTART/20200825.180000.fv_tracer.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_tracer.res.tile1.nc \
RESTART/20200825.180000.fv_tracer.res.tile2.nc \
RESTART/20200825.180000.fv_tracer.res.tile3.nc \
RESTART/20200825.180000.fv_tracer.res.tile4.nc \
RESTART/20200825.180000.fv_tracer.res.tile5.nc \
RESTART/20200825.180000.fv_tracer.res.tile6.nc \
RESTART/20200825.180000.phy_data.nest02.tile7.nc \
RESTART/20200825.180000.phy_data.tile1.nc \
RESTART/20200825.180000.phy_data.tile2.nc \
RESTART/20200825.180000.phy_data.tile3.nc \
RESTART/20200825.180000.phy_data.tile4.nc \
RESTART/20200825.180000.phy_data.tile5.nc \
RESTART/20200825.180000.phy_data.tile6.nc \
RESTART/20200825.180000.sfc_data.nest02.tile7.nc \
RESTART/20200825.180000.sfc_data.tile1.nc \
RESTART/20200825.180000.sfc_data.tile2.nc \
RESTART/20200825.180000.sfc_data.tile3.nc \
RESTART/20200825.180000.sfc_data.tile4.nc \
RESTART/20200825.180000.sfc_data.tile5.nc \
RESTART/20200825.180000.sfc_data.tile6.nc \
RESTART/fv_BC_ne.res.nest02.nc \
RESTART/fv_BC_sw.res.nest02.nc"


export_fv3
export_hafs
Expand Down
155 changes: 155 additions & 0 deletions tests/tests/hafs_global_1nest_atm_qr
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
###############################################################################
#
# HAFS global with 1 nest and atmosphere only
#
###############################################################################

export TEST_DESCR="Compare HAFS global with 1 nest and atmosphere only results with previous trunk version"

export CNTL_DIR=hafs_global_1nest_atm

export LIST_FILES="atmf006.nc \
sfcf006.nc \
atm.nest02.f006.nc \
sfc.nest02.f006.nc \
RESTART/20200825.180000.coupler.res \
RESTART/20200825.180000.fv_core.res.nc \
RESTART/20200825.180000.fv_core.res.nest02.nc \
RESTART/20200825.180000.fv_core.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_core.res.tile1.nc \
RESTART/20200825.180000.fv_core.res.tile2.nc \
RESTART/20200825.180000.fv_core.res.tile3.nc \
RESTART/20200825.180000.fv_core.res.tile4.nc \
RESTART/20200825.180000.fv_core.res.tile5.nc \
RESTART/20200825.180000.fv_core.res.tile6.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile1.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile2.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile3.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile4.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile5.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile6.nc \
RESTART/20200825.180000.fv_tracer.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_tracer.res.tile1.nc \
RESTART/20200825.180000.fv_tracer.res.tile2.nc \
RESTART/20200825.180000.fv_tracer.res.tile3.nc \
RESTART/20200825.180000.fv_tracer.res.tile4.nc \
RESTART/20200825.180000.fv_tracer.res.tile5.nc \
RESTART/20200825.180000.fv_tracer.res.tile6.nc \
RESTART/20200825.180000.phy_data.nest02.tile7.nc \
RESTART/20200825.180000.phy_data.tile1.nc \
RESTART/20200825.180000.phy_data.tile2.nc \
RESTART/20200825.180000.phy_data.tile3.nc \
RESTART/20200825.180000.phy_data.tile4.nc \
RESTART/20200825.180000.phy_data.tile5.nc \
RESTART/20200825.180000.phy_data.tile6.nc \
RESTART/20200825.180000.sfc_data.nest02.tile7.nc \
RESTART/20200825.180000.sfc_data.tile1.nc \
RESTART/20200825.180000.sfc_data.tile2.nc \
RESTART/20200825.180000.sfc_data.tile3.nc \
RESTART/20200825.180000.sfc_data.tile4.nc \
RESTART/20200825.180000.sfc_data.tile5.nc \
RESTART/20200825.180000.sfc_data.tile6.nc \
RESTART/fv_BC_ne.res.nest02.nc \
RESTART/fv_BC_sw.res.nest02.nc"

export_fv3
export_hafs

export HAFS=true
export FHMAX=6
export RESTART_N=${FHMAX}
export atm_omp_num_threads=2
export DT_ATMOS=90
export IDEFLATE=1
export NFHMAX_HF=-1
export NFHOUT_HF=3
export NFHOUT=3
export OUTPUT_FILE="'netcdf' 'netcdf'"
export SDAY=25
export SHOUR=12
export SMONTH=08
export SYEAR=2020
export WRTTASK_PER_GROUP=60
export NTILES=6

export QUILTING_RESTART=.true.
export OUTPUT_GRID='global_latlon'
export IMO=360
export JMO=181

export OUTPUT_GRID_2='rotated_latlon'
export CEN_LON_2=-86.0
export CEN_LAT_2=23.0
export LON1_2=-45.00
export LAT1_2=-40.00
export LON2_2=45.00
export LAT2_2=40.00
export DLON_2=0.25
export DLAT_2=0.25

export BLOCKSIZE=${BLOCKSIZE:-24}
export CCPP_SUITE="FV3_HAFS_v1_gfdlmp_tedmf"
export DEFLATE_LEVEL=-1
export INPES=4
export JNPES=5
export NPX=97
export NPY=97
export NPZ=64
export NPZP=$(($NPZ + 1))
export K_SPLIT=4
export N_SPLIT=5
export NWAT=6
export NA_INIT=1
export EXTERNAL_IC=.true.
export NGGPS_IC=.true.
export MOUNTAIN=.false.
export WARM_START=.false.
export TARGET_LAT="23.0"
export TARGET_LON="-86.0"
export STRETCH_FAC=1.0001
export FULL_ZS_FILTER=.false.
export N_ZS_FILTER=0

export GRID_PES="120,60"
export TILE_COARSE="0,6"
export NUM_TILE_TOP=6
export NEST_REFINE="0,3"
export NEST_IOFFSETS="1,9"
export NEST_JOFFSETS="1,9"

export FHCYC=0
export FHSWR=1800
export FHLWR=1800
export NSTF_NAME=2,0,0,0,0
export CPLFLX=.false.
export CPLOCN2ATM=.true.
export CPLWAV=.false.
export CPLWAV2ATM=.false.
export CPL_IMP_MRG=.false.

export INPES_NEST02=6
export JNPES_NEST02=10
export NPX_NEST02=241
export NPY_NEST02=241
export K_SPLIT_NEST02=4
export N_SPLIT_NEST02=5

ATM_compute_tasks=$(( INPES*JNPES*NTILES + INPES_NEST02*JNPES_NEST02 ))

export NESTED=.true.
export FULL_ZS_FILTER=.false.
export N_ZS_FILTER=0

export CMEPS=false
export RUNTYPE="startup"
export USE_COLDSTART="false"

export FIELD_TABLE=field_table_hafs
export DIAG_TABLE=diag_table_hafs_template
export INPUT_NML=input_global_hafs.nml.IN
export INPUT_NEST02_NML=input_nest_hafs.nml.IN
export MODEL_CONFIGURE="model_configure_hafs.IN"
export NEMS_CONFIGURE="nems.configure.hafs_atm.IN"
export FV3_RUN="hafs_fv3_run.IN"

66 changes: 65 additions & 1 deletion tests/tests/hafs_global_multiple_4nests_atm
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,71 @@ export LIST_FILES="atmf006.nc \
HURPRS.GrbF06.nest02 \
HURPRS.GrbF06.nest03 \
HURPRS.GrbF06.nest04 \
HURPRS.GrbF06.nest05"
HURPRS.GrbF06.nest05 \
RESTART/20200825.180000.coupler.res \
RESTART/20200825.180000.fv_core.res.nc \
RESTART/20200825.180000.fv_core.res.nest02.nc \
RESTART/20200825.180000.fv_core.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_core.res.nest03.nc \
RESTART/20200825.180000.fv_core.res.nest03.tile8.nc \
RESTART/20200825.180000.fv_core.res.nest04.nc \
RESTART/20200825.180000.fv_core.res.nest04.tile9.nc \
RESTART/20200825.180000.fv_core.res.nest05.nc \
RESTART/20200825.180000.fv_core.res.nest05.tile10.nc \
RESTART/20200825.180000.fv_core.res.tile1.nc \
RESTART/20200825.180000.fv_core.res.tile2.nc \
RESTART/20200825.180000.fv_core.res.tile3.nc \
RESTART/20200825.180000.fv_core.res.tile4.nc \
RESTART/20200825.180000.fv_core.res.tile5.nc \
RESTART/20200825.180000.fv_core.res.tile6.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest03.tile8.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest04.tile9.nc \
RESTART/20200825.180000.fv_srf_wnd.res.nest05.tile10.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile1.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile2.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile3.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile4.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile5.nc \
RESTART/20200825.180000.fv_srf_wnd.res.tile6.nc \
RESTART/20200825.180000.fv_tracer.res.nest02.tile7.nc \
RESTART/20200825.180000.fv_tracer.res.nest03.tile8.nc \
RESTART/20200825.180000.fv_tracer.res.nest04.tile9.nc \
RESTART/20200825.180000.fv_tracer.res.nest05.tile10.nc \
RESTART/20200825.180000.fv_tracer.res.tile1.nc \
RESTART/20200825.180000.fv_tracer.res.tile2.nc \
RESTART/20200825.180000.fv_tracer.res.tile3.nc \
RESTART/20200825.180000.fv_tracer.res.tile4.nc \
RESTART/20200825.180000.fv_tracer.res.tile5.nc \
RESTART/20200825.180000.fv_tracer.res.tile6.nc \
RESTART/20200825.180000.phy_data.nest02.tile7.nc \
RESTART/20200825.180000.phy_data.nest03.tile8.nc \
RESTART/20200825.180000.phy_data.nest04.tile9.nc \
RESTART/20200825.180000.phy_data.nest05.tile10.nc \
RESTART/20200825.180000.phy_data.tile1.nc \
RESTART/20200825.180000.phy_data.tile2.nc \
RESTART/20200825.180000.phy_data.tile3.nc \
RESTART/20200825.180000.phy_data.tile4.nc \
RESTART/20200825.180000.phy_data.tile5.nc \
RESTART/20200825.180000.phy_data.tile6.nc \
RESTART/20200825.180000.sfc_data.nest02.tile7.nc \
RESTART/20200825.180000.sfc_data.nest03.tile8.nc \
RESTART/20200825.180000.sfc_data.nest04.tile9.nc \
RESTART/20200825.180000.sfc_data.nest05.tile10.nc \
RESTART/20200825.180000.sfc_data.tile1.nc \
RESTART/20200825.180000.sfc_data.tile2.nc \
RESTART/20200825.180000.sfc_data.tile3.nc \
RESTART/20200825.180000.sfc_data.tile4.nc \
RESTART/20200825.180000.sfc_data.tile5.nc \
RESTART/20200825.180000.sfc_data.tile6.nc \
RESTART/fv_BC_ne.res.nest02.nc \
RESTART/fv_BC_ne.res.nest03.nc \
RESTART/fv_BC_ne.res.nest04.nc \
RESTART/fv_BC_ne.res.nest05.nc \
RESTART/fv_BC_sw.res.nest02.nc \
RESTART/fv_BC_sw.res.nest03.nc \
RESTART/fv_BC_sw.res.nest04.nc \
RESTART/fv_BC_sw.res.nest05.nc"

export_fv3
export_hafs
Expand Down
Loading

0 comments on commit 52cee26

Please sign in to comment.