diff --git a/.gitmodules b/.gitmodules index 71f29012..b43d0a53 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ [submodule "DA_update"] path = DA_update url = https://github.com/ufs-community/land-DA.git - branch = develop + branch = feature/porting [submodule "vector2tile"] path = vector2tile url = https://github.com/NOAA-PSL/land-vector2tile.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 03cae5a8..49a26108 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ ExternalProject_Add(ufs-weather-model ) add_subdirectory( vector2tile ) +add_subdirectory( tile2tile ) add_subdirectory( ufs-land-driver-emc-dev ) add_subdirectory( DA_update/add_jedi_incr ) diff --git a/DA_update b/DA_update index e4171946..a81c8fd5 160000 --- a/DA_update +++ b/DA_update @@ -1 +1 @@ -Subproject commit e4171946e2dd6bb1829a8bba9198486df02815bb +Subproject commit a81c8fd5ec4a6c571f76ce3426f2b0777a6d2349 diff --git a/datm_cdeps_lnd_gswp3_rst b/datm_cdeps_lnd_gswp3_rst new file mode 100644 index 00000000..e001ad37 --- /dev/null +++ b/datm_cdeps_lnd_gswp3_rst @@ -0,0 +1,62 @@ +# +# DATM_CDEPS_NOAHMP_GSWP test +# + +export TEST_DESCR="DATM_CDEPS_NOAHMP_GSWP3_RST - control restart" + +export CNTL_DIR="datm_cdeps_lnd_gswp3" + +export LIST_FILES="ufs.cpld.lnd.out.2000-01-02-00000.tile1.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile2.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile3.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile4.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile5.nc \ + ufs.cpld.lnd.out.2000-01-02-00000.tile6.nc" + +export_datm_cdeps + +export SYEAR=2000 +export SMONTH=01 +export SDAY=03 +export SHOUR=00 +export FHMAX=24 +export FHROT=0 + +export DATM_IN_CONFIGURE=datm_in +export MESH_ATM="INPUT_DATM/fv1.9x2.5_141008_ESMFmesh.nc" +export atm_datamode="CLMNCEP" +export ATM_NX_GLB=144 +export ATM_NY_GLB=96 + +export DATM_STREAM_CONFIGURE=datm.streams.multi.IN +export MESH_ATM_DATA="INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.SCRIP.210520_ESMFmesh.nc" +export MESH_ATM_TOPO="INPUT_DATM/topodata_0.9x1.SCRIP.210520_ESMFmesh.nc" +export SDATE=1999 +export EDATE=2000 +export DATA_ATM01="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Solr.2000-01.nc\"" +export DATA_ATM02="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.Prec.2000-01.nc\"" +export DATA_ATM03="\"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.1999-12.nc\" \"INPUT_DATM/clmforc.GSWP3.c2011.0.5x0.5.TPQWL.2000-01.nc\"" +export DATA_ATM04="\"INPUT_DATM/topodata_0.9x1.25_USGS_070110_stream_c151201.nc\"" + +export UFS_CONFIGURE="ufs.configure.atm_lnd.IN" +export med_model="cmeps" +export atm_model="datm" +export lnd_model="noahmp" +export CPLMODE="ufs.nfrac.aoflux" +export RUNTYPE="continue" +export READRESTART=.false. +export ATM_compute_tasks=144 +export OCN_tasks=0 +export ICE_tasks=0 +export LND_tasks=144 +export lnd_ic_type='custom' +export CALC_SNET=.true. +export layout_x=3 +export layout_y=8 +export mosaic_file="INPUT/C96_mosaic.nc" +export coupling_interval_sec=3600 +export TOPOEDITS="" +export WARM_START=.true. + +export RESTART_FILE_SUFFIX_SECS="${SYEAR}-${SMONTH}-${SDAY}-$(printf "%05d" $(( (${FHROT}+ ${SHOUR})*3600 )))" +export FV3_RUN="lnd_datm_cdeps_gswp.IN noahmp_run.IN" diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh index 1954b551..a40174c8 100755 --- a/do_submit_cycle.sh +++ b/do_submit_cycle.sh @@ -20,7 +20,6 @@ source $config_file export KEEPWORKDIR="YES" - ############################ # ensure necessary envars are set envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "CYCLEDIR" \ @@ -77,9 +76,14 @@ fi if [[ -e ${BUILDDIR}/bin/vector2tile_converter.exe ]]; then #prefer cmake-built executables export vec2tileexec=${BUILDDIR}/bin/vector2tile_converter.exe -else +else export vec2tileexec=${CYCLEDIR}/vector2tile/vector2tile_converter.exe fi +if [[ -e ${BUILDDIR}/bin/tile2tile_converter.exe ]]; then #prefer cmake-built executables + export tile2tileexec=${BUILDDIR}/bin/tile2tile_converter.exe +else + export tile2tileexec=${CYCLEDIR}/tile2tile/tile2tile_converter.exe +fi if [[ -e ${BUILDDIR}/bin/ufsLand.exe ]]; then export LSMexec=${BUILDDIR}/bin/ufsLand.exe else @@ -154,20 +158,42 @@ fi ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output # copy ICS into restarts, if needed -rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc -rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc -# if restart not in experiment out directory, copy the restarts from the ICSDIR -if [[ ! -e ${rst_out} ]]; then - echo "Looking for ICS: ${rst_in}" - if [[ -e ${rst_in} ]]; then - echo "ICS found, copying" - cp ${rst_in} ${rst_out} - else - echo "ICS not found. Exiting" - exit 10 +if [[ $atmos_forc == "era5" ]]; then + rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc + # if restart not in experiment out directory, copy the restarts from the ICSDIR + if [[ ! -e ${rst_out} ]]; then + echo "Looking for ICS: ${rst_in}" + if [[ -e ${rst_in} ]]; then + echo "ICS found, copying" + cp ${rst_in} ${rst_out} + else + echo "ICS not found. Exiting" + exit 10 + fi fi fi +if [[ $atmos_forc == "gswp3" ]]; then + + for tile in 1 2 3 4 5 6 + do + rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${sYYYY}-${sMM}-${sDD}-00000.tile${tile}.nc + # if restart not in experiment out directory, copy the restarts from the ICSDIR + if [[ ! -e ${rst_out} ]]; then + echo "Looking for ICS: ${rst_in}" + if [[ -e ${rst_in} ]]; then + echo "ICS found, copying" + cp ${rst_in} ${rst_out} + else + echo "ICS not found. Exiting" + exit 10 + fi + fi + done +fi + # create dates file touch analdates.sh cat << EOF > analdates.sh diff --git a/modulefiles/landda_orion.intel.lua b/modulefiles/landda_orion.intel.lua index f4785001..bb515c04 100644 --- a/modulefiles/landda_orion.intel.lua +++ b/modulefiles/landda_orion.intel.lua @@ -2,7 +2,7 @@ help([[ loads UFS Model prerequisites for Orion/Intel ]]) -prepend_path("MODULEPATH", "/work/noaa/epic-ps/role-epic-ps/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles") stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" diff --git a/settings_DA_cycle_era5 b/settings_DA_cycle_era5 index edf5721b..a5c68dc0 100644 --- a/settings_DA_cycle_era5 +++ b/settings_DA_cycle_era5 @@ -1,7 +1,7 @@ # Settings file for submit_cycle, for running the DA_GHCN_test # experiment name -export exp_name=DA_GHCN_test +export exp_name=DA_ERA5_test #export BASELINE=hera.internal #ERA5 forcing is available for 2020-2021 @@ -12,7 +12,7 @@ ENDDATE=2019122200 source ./release.environment ############################ -#forcing options: gdas, era5 +#forcing options: gswp3, era5 export atmos_forc=era5 # for LETKF, this is size of ensemble. diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3 new file mode 100644 index 00000000..d77cb54d --- /dev/null +++ b/settings_DA_cycle_gswp3 @@ -0,0 +1,53 @@ +# Settings file for submit_cycle, for running the DA_GHCN_test + +# experiment name +export exp_name=DA_GSWP3_test +#export BASELINE=hera.internal + +#Machine options: orion, hera +export MACHINE_ID=hera + +# experiment period +STARTDATE=2000010300 +ENDDATE=2000010400 + +# Get commmon variables +source ./release.environment +############################ + +#forcing options: gswp3, era5 +export atmos_forc=gswp3 + +# for LETKF, this is size of ensemble. +# for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 +export ensemble_size=1 + +# length of each forecast +export FCSTHR=24 + +#FV3 resolution +export RES=96 +if [[ $BASELINE =~ 'hera.internal' ]]; then + export TPATH=/scratch2/NCEPDEV/land/data/fix/C96.mx100_frac/ +else + export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/" +fi +export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH + # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean. + +# number of cycles to submit in a single job +export cycles_per_job=1 + +# directory with initial conditions +# can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single +export ICSDIR=$LANDDAROOT/inputs/forcing/${atmos_forc}/orog_files/ + +# namelist for do_landDA.sh +# set to "openloop" to not call do_landDA.sh +export DA_config="settings_DA_test" + +# if want different DA at different times, list here. +export DA_config00=${DA_config} +export DA_config06=${DA_config} +export DA_config12=${DA_config} +export DA_config18=${DA_config} diff --git a/submit_cycle.sh b/submit_cycle.sh index c041aaa1..a1157a46 100755 --- a/submit_cycle.sh +++ b/submit_cycle.sh @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --job-name=offline_noahmp +#SBATCH --job-name=ufs_land_da #SBATCH --account=da-cpu #SBATCH --qos=debug #SBATCH --nodes=1 @@ -58,7 +58,7 @@ while [ $date_count -lt $cycles_per_job ]; do nHH=`echo $NEXTDATE | cut -c9-10` ############################ - # copy restarts to workdir, convert to vector for DA (all members) + # copy restarts to workdir, convert to UFS tile for DA (all members) mem_ens="mem000" @@ -66,17 +66,19 @@ while [ $date_count -lt $cycles_per_job ]; do MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} cd $MEM_WORKDIR - # copy restarts into work directory - rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - cp ${rst_in} ${rst_out} - if [[ $do_jedi == "YES" ]]; then + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then + + # copy restarts into work directory + rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + cp ${rst_in} ${rst_out} + echo '************************************************' - echo 'calling tile2vector' + echo 'calling vector2tile' export MEM_WORKDIR @@ -97,17 +99,62 @@ while [ $date_count -lt $cycles_per_job ]; do # submit vec2tile echo '************************************************' echo 'calling vector2tile' - + if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods - fi + fi $vec2tileexec vector2tile.namelist if [[ $? != 0 ]]; then echo "vec2tile failed" - exit + exit fi fi # vector2tile for DA + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then + + echo '************************************************' + echo 'calling tile2tile' + + export MEM_WORKDIR + + # copy restarts into work directory + + for tile in 1 2 3 4 5 6 + do + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${rst_in} ${rst_out} + done + + # update tile2tile namelist + cp ${CYCLEDIR}/template.ufs2jedi ufs2jedi.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist + sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist + sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" ufs2jedi.namelist + sed -i -e "s/XXRES/${RES}/g" ufs2jedi.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist + + # submit tile2tile + + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + $tile2tileexec ufs2jedi.namelist + if [[ $? != 0 ]]; then + echo "tile2tile failed" + exit + fi + fi # tile2tile for DA + ############################ # do DA update @@ -126,9 +173,9 @@ while [ $date_count -lt $cycles_per_job ]; do exit fi fi - + ############################ - # convert back to vector, run model (all members) convert back to vector, run model (all members) + # convert back to vector/tile mem_ens="mem000" @@ -137,16 +184,17 @@ while [ $date_count -lt $cycles_per_job ]; do cd $MEM_WORKDIR - if [[ $do_jedi == "YES" ]]; then + # convert back to vector, run model (all members) convert back to vector, run model (all members) + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then echo '************************************************' echo 'calling tile2vector' - + if [[ $BASELINE =~ 'hera.internal' ]]; then source ${CYCLEDIR}/land_mods - fi - + fi + cp ${CYCLEDIR}/template.tile2vector tile2vector.namelist - + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist @@ -160,62 +208,207 @@ while [ $date_count -lt $cycles_per_job ]; do $vec2tileexec tile2vector.namelist if [[ $? != 0 ]]; then echo "tile2vector failed" - exit + exit fi # save analysis restart cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc fi + # convert back to UFS tile, run model (all members) + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then + echo '************************************************' + echo 'calling tile2tile' + + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + + cp ${CYCLEDIR}/template.jedi2ufs jedi2ufs.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist + sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist + sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist + sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist + sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" jedi2ufs.namelist + sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist + + $tile2tileexec jedi2ufs.namelist + if [[ $? != 0 ]]; then + echo "tile2tile failed" + exit + fi + + # save analysis restart + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + done + fi + ############################ # run the forecast model - set -x - # update model namelist - cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist - sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist - sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist - sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist - sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist - sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist - sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist - - # submit model - echo '************************************************' - echo "calling model" - echo $MEM_WORKDIR - - nt=$SLURM_NTASKS - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${CYCLEDIR}/land_mods + + if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then + echo '************************************************' + echo 'running the forecast model' + + set -x + + # update model namelist + cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist + sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist + sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist + sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist + sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist + sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist + sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist + + # submit model + echo $MEM_WORKDIR + + nt=$SLURM_NTASKS + if [[ $BASELINE =~ 'hera.internal' ]]; then + source ${CYCLEDIR}/land_mods + fi + + if [[ $BASELINE =~ 'hera.internal' ]]; then + srun '--export=ALL' --label -K -n $nt $LSMexec + else + ${MPIEXEC} -n 1 $LSMexec + fi fi + # no error codes on exit from model, check for restart below instead + + if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then + set -e + + echo '************************************************' + echo 'running the forecast model' - if [[ $BASELINE =~ 'hera.internal' ]]; then - srun '--export=ALL' --label -K -n $nt $LSMexec - else - ${MPIEXEC} -n 1 $LSMexec - fi + TEST_NAME=datm_cdeps_lnd_gswp3 + TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst + PATHRT=${CYCLEDIR}/ufs-weather-model/tests + RT_COMPILER=${RT_COMPILER:-intel} + ATOL="1e-7" + + cp $CYCLEDIR/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/detect_machine.sh + source ${PATHRT}/rt_utils.sh + source ${PATHRT}/default_vars.sh + source ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/atparse.bash + + # Set inputdata location for each machines + echo "MACHINE_ID: $MACHINE_ID" + if [[ $MACHINE_ID = orion ]]; then + DISKNM=/work/noaa/epic/UFS-WM_RT + elif [[ $MACHINE_ID = hera ]]; then + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + else + echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" + fi + + source ${PATHRT}/bl_date.conf + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} + + echo "RTPWD= $RTPWD" + echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" + + if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 + fi + + # create run folder + RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} + [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} + mkdir -p ${RUNDIR} + cd ${RUNDIR} + + echo "NoahMP run dir= $RUNDIR" + + # modify some env variables - reduce core usage + export ATM_compute_tasks=0 + export ATM_io_tasks=1 + export LND_tasks=6 + export layout_x=1 + export layout_y=1 + + # FV3 executable: + cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model + cp ${LANDDA_INPUTS}/restarts/fv3_run ./fv3_run + + if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then + if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then + atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml + fi + fi + + atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure + + compute_petbounds_and_tasks + + atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure + + # diag table + if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then + atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table + fi + + # Field table + if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then + cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table + fi + + # Field Dictionary + cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml + + # Set up the run directory + source ./fv3_run + + if [[ $DATM_CDEPS = 'true' ]]; then + atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in} > datm_in + atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams + fi + + # NoahMP table file + cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl + + # start runs + echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" + export MPIRUN=${MPIRUN:-`which mpiexec`} + ${MPIRUN} -n ${TASKS} ./ufs_model + fi + # no error codes on exit from model, check for restart below instead ############################ # check model ouput (all members) - mem_ens="mem000" + mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - - if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + + if [[ $atmos_forc == "era5" ]]; then + # if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc - else - echo "Something is wrong, probably the model, exiting" - exit + else + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + done fi - -# echo "Finished job number, ${date_count},for date: ${THISDATE}" >> $logfile - + THISDATE=$NEXTDATE date_count=$((date_count+1)) diff --git a/template.jedi2ufs b/template.jedi2ufs new file mode 100644 index 00000000..08cd7ffd --- /dev/null +++ b/template.jedi2ufs @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "ufs2jedi" or "jedi2ufs" for restart file +! + + direction = "jedi2ufs" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + +! tile_restart_path = "junk" + tile_restart_path = "./" +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ diff --git a/template.ufs2jedi b/template.ufs2jedi new file mode 100644 index 00000000..b5028419 --- /dev/null +++ b/template.ufs2jedi @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "ufs2jedi" or "jedi2ufs" for restart file +! + + direction = "ufs2jedi" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = XXRES + tile_path = "XXTPATH" + tile_fstub = "XXTSTUB" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + +! Path for static file + static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "./" + +! Location of tile restart files (tile2vector direction) + +! tile_restart_path = "junk" + tile_restart_path = "./" +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "" + +! input perturbation pattern files + + lndp_input_file = "" + +! output files + + lndp_output_file = "" + +! land perturbation variable list, for an example: 'vgf','smc' + + lndp_var_list = '' + +/ diff --git a/test/ci/Jenkinsfile b/test/ci/Jenkinsfile index d1e456c7..55dac041 100644 --- a/test/ci/Jenkinsfile +++ b/test/ci/Jenkinsfile @@ -1,6 +1,16 @@ pipeline { agent none stages { + stage('Launch SonarQube') { + steps { + script { + build job: '/land-da/land-da-sonarqube', parameters: [ + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: '') + ], wait: false + } + } + } stage('Land DA workflow') { agent { label 'built-in' @@ -51,7 +61,7 @@ pipeline { } environment { ACCNR = 'nems' - NODE_PATH = '/scratch1/NCEPDEV/stmp2/role.epic/' + NODE_PATH = '/scratch2/NAGAPE/epic/role.epic/' } steps { @@ -59,17 +69,17 @@ pipeline { checkout scm sh ''' git submodule update --init --recursive - ln -fs /scratch2/NAGAPE/epic/UFS_Land-DA/inputs /scratch1/NCEPDEV/stmp2/role.epic/jenkins/workspace/ + ln -fs /scratch2/NAGAPE/epic/UFS_Land-DA/inputs /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/ module use modulefiles && module load landda_hera.intel mkdir build cp test/hera_ctest.sh build/ cp test/check_ctest.sh build/ - cp test/run_ctest.sh build/ + cp test/run_hera_ctest.sh build/ cd build/ module use ../modulefiles && module load landda_hera.intel ecbuild ../ && make -j4 echo $(pwd) - ./run_ctest.sh + ./run_hera_ctest.sh ''' } } @@ -86,7 +96,21 @@ pipeline { cleanWs() checkout scm sh ''' - + git submodule update --init --recursive + ln -fs /work/noaa/epic/UFS_Land-DA/inputs /work/noaa/epic/role-epic/jenkins/workspace/ + module use modulefiles && module load landda_orion.intel.lua + mkdir build + cp test/orion_ctest.sh build/ + cp test/check_ctest.sh build/ + cp test/run_orion_ctest.sh build/ + cd test/ + sed -i 's|MACHINE_ID=${MACHINE_ID:-hera}|MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh + cd .. + cd build/ + module use ../modulefiles && module load landda_orion.intel.lua + ecbuild ../ && make -j4 + echo $(pwd) + ctest ''' } } diff --git a/test/orion_ctest.sh b/test/orion_ctest.sh new file mode 100755 index 00000000..637de305 --- /dev/null +++ b/test/orion_ctest.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH -o out.ctest +#SBATCH --account=nems +set -eux + + module use modulefiles && module load landda_orion.intel.lua + +ctest + +wait + +echo "ctest is done" diff --git a/test/run_ctest.sh b/test/run_hera_ctest.sh similarity index 100% rename from test/run_ctest.sh rename to test/run_hera_ctest.sh diff --git a/test/run_orion_ctest.sh b/test/run_orion_ctest.sh new file mode 100755 index 00000000..dfc12290 --- /dev/null +++ b/test/run_orion_ctest.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux + +JOB_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=6 --nodes=1 --time=00:30:00 ./orion_ctest.sh | awk '{print $4}') + +CHECK_ID=$(sbatch --job-name=ctest --account=epic --qos=debug --ntasks-per-node=1 --nodes=1 --time=00:01:00 --dependency=afterok:$JOB_ID ./check_ctest.sh) + +sleep 3m + +if [ -f out.ctest ]; then + cat out.ctest +else + echo "ctest run fails to run." +fi diff --git a/test/run_ufs_datm_lnd.sh b/test/run_ufs_datm_lnd.sh index 6f0056a6..151fe650 100755 --- a/test/run_ufs_datm_lnd.sh +++ b/test/run_ufs_datm_lnd.sh @@ -10,7 +10,7 @@ echo ${project_binary_dir} echo ${project_source_dir} # -export MACHINE_ID=${MACHINE_ID:-hera} +MACHINE_ID=${MACHINE_ID:-hera} TEST_NAME=datm_cdeps_lnd_gswp3 PATHRT=${project_source_dir}/ufs-weather-model/tests RT_COMPILER=${RT_COMPILER:-intel} @@ -24,15 +24,15 @@ source ${PATHRT}/atparse.bash # Set inputdata location for each machines echo "MACHINE_ID: $MACHINE_ID" if [[ $MACHINE_ID = orion ]]; then - DISKNM=/work/noaa/nems/emc.nemspara/RT + DISKNM=/work/noaa/epic/UFS-WM_RT elif [[ $MACHINE_ID = hera ]]; then - DISKNM=/scratch1/NCEPDEV/nems/emc.nemspara/RT + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT else echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves" fi -#source ${PATHRT}/bl_date.conf -BL_DATE=20230815 +source ${PATHRT}/bl_date.conf +#BL_DATE=20230815 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}} INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} @@ -73,7 +73,7 @@ atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configur compute_petbounds_and_tasks -atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure +atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure # diag table if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then @@ -85,7 +85,7 @@ if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then fi # Field Dictionary -cp ${PATHRT}/parm/fd_nems.yaml fd_nems.yaml +cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml # Set up the run directory source ./fv3_run diff --git a/tile2tile/.gitignore b/tile2tile/.gitignore new file mode 100644 index 00000000..18de9433 --- /dev/null +++ b/tile2tile/.gitignore @@ -0,0 +1,4 @@ +vector2tile_converter.exe +vector2tile_converter.o +*.mod +*.o diff --git a/tile2tile/CMakeLists.txt b/tile2tile/CMakeLists.txt new file mode 100644 index 00000000..5e52d289 --- /dev/null +++ b/tile2tile/CMakeLists.txt @@ -0,0 +1,75 @@ +# (C) Copyright 2022 . +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +cmake_minimum_required( VERSION 3.12) +project( tile2tile VERSION 2022.10 LANGUAGES Fortran ) + +# ecbuild integration +find_package(ecbuild 3.3.2 REQUIRED) +include( ecbuild_system NO_POLICY_SCOPE ) +ecbuild_declare_project() + +list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +#include( tile2tile_compiler_flags ) + +################################################################################ +# Dependencies +################################################################################ + +find_package(OpenMP COMPONENTS C Fortran) +find_package(MPI REQUIRED COMPONENTS C Fortran) +find_package(NetCDF REQUIRED COMPONENTS Fortran ) + +################################################################################ +# Sources +################################################################################ + +set( TILE2TILE_LINKER_LANGUAGE Fortran ) + +list ( APPEND src_files +./namelist_mod.f90 +./tile2tile_restart_mod.f90 +) + +ecbuild_add_library( TARGET tile2tile + SOURCES ${src_files} + INSTALL_HEADERS LISTED + LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE} + ) + + +target_link_libraries(tile2tile PUBLIC NetCDF::NetCDF_Fortran) +target_link_libraries(tile2tile PUBLIC MPI::MPI_Fortran) +target_link_libraries(tile2tile PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_Fortran) + +# Fortran module output directory for build and install interfaces +set(MODULE_DIR module/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}) +set_target_properties(${PROJECT_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}) +install(DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}/ DESTINATION ${MODULE_DIR}) +target_include_directories(${PROJECT_NAME} INTERFACE + $ + $) + +set( TILE2TILE_INCLUDE_DIRS ${CMAKE_Fortran_MODULE_DIRECTORY} ) +set( TILE2TILE_LIBRARIES tile2tile ) + +# tile2tile_converter.exe executable +#------------------------------------------------------------------------------- +set ( exe_files tile2tile_driver.f90 ) + +ecbuild_add_executable( TARGET tile2tile_converter.exe + SOURCES ${exe_files} + LIBS tile2tile + LINKER_LANGUAGE ${TILE2TILE_LINKER_LANGUAGE} + ) + + +################################################################################ +# Finalise configuration +################################################################################ + +ecbuild_install_project( NAME tile2tile ) +ecbuild_print_summary() diff --git a/tile2tile/Makefile b/tile2tile/Makefile new file mode 100644 index 00000000..45308c7e --- /dev/null +++ b/tile2tile/Makefile @@ -0,0 +1,24 @@ +# Makefile +# +.SUFFIXES: +.SUFFIXES: .o .f90 + +include ./user_build_config + +OBJS = namelist_mod.o vector2tile_restart_mod.o vector2tile_perturbation_mod.o vector2tile_driver.o + +all: vector2tile_converter.exe + +.f90.o: + $(COMPILERF90) -c $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(*).f90 + +vector2tile_converter.exe: $(OBJS) + $(COMPILERF90) -o $(@) $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(OBJS) $(NETCDFLIB) + +clean: + rm -f *.o *.mod *.exe + + +# +# Dependencies: +# diff --git a/tile2tile/cmake/compiler_flags_GNU_CXX.cmake b/tile2tile/cmake/compiler_flags_GNU_CXX.cmake new file mode 100755 index 00000000..dbe13c02 --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_CXX.cmake @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +if( HAVE_OMP ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fopenmp") +else( ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fno-openmp") +endif( ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_DEBUG "-O0 -g" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_BIT "-O2" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_CXX_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake new file mode 100755 index 00000000..c0b618cc --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake @@ -0,0 +1,43 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test new file mode 100755 index 00000000..95f6209b --- /dev/null +++ b/tile2tile/cmake/compiler_flags_GNU_Fortran.cmake_test @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -fbacktrace -ffree-line-length-none -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -ffree-form") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) +set( CMAKE_Fortran_FLAGS_RELEASE " " ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake b/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake new file mode 100755 index 00000000..7eef4052 --- /dev/null +++ b/tile2tile/cmake/compiler_flags_Intel_Fortran.cmake @@ -0,0 +1,41 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -convert big_endian -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -sox -traceback -msse2" ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3 -debug minimal -fp-model source" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -debug minimal -fp-model source" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/tile2tile/cmake/vector2tile_compiler_flags.cmake b/tile2tile/cmake/vector2tile_compiler_flags.cmake new file mode 100755 index 00000000..8aebfaec --- /dev/null +++ b/tile2tile/cmake/vector2tile_compiler_flags.cmake @@ -0,0 +1,23 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#add_definitions ( -Duse_libMPI -Duse_netCDF -DSPMD -DNXGLOB=4) +add_definitions ( -Duse_libMPI -Duse_netCDF -Dncdf -Dgather_scatter_barrier) + +####################################################################################### +# Fortran +####################################################################################### + +if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) + include( compiler_flags_GNU_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) + include( compiler_flags_Intel_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "XL" ) + include( compiler_flags_XL_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" ) + include( compiler_flags_Cray_Fortran ) +else() + message( STATUS "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options") +endif() diff --git a/tile2tile/config/user_build_config.gfortran.serial b/tile2tile/config/user_build_config.gfortran.serial new file mode 100644 index 00000000..a7191d5c --- /dev/null +++ b/tile2tile/config/user_build_config.gfortran.serial @@ -0,0 +1,11 @@ + +#=============================================================================== +# Placeholder options for Mac gfortran +#=============================================================================== + + COMPILERF90 = /opt/local/bin/gfortran-mp-11 + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/opt/local/include + NETCDFLIB = -L/opt/local/lib -lnetcdf -lnetcdff + diff --git a/tile2tile/config/user_build_config.hera b/tile2tile/config/user_build_config.hera new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/tile2tile/config/user_build_config.hera @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/tile2tile/configure b/tile2tile/configure new file mode 100755 index 00000000..24aad730 --- /dev/null +++ b/tile2tile/configure @@ -0,0 +1,39 @@ +#!/usr/bin/perl + + if($#ARGV == 0) { + $response = shift(@ARGV) ; + print("Configure soil test environment: $response \n"); + }else { + print "Please select from following predefined environments: \n\n"; + + print " 1. hera \n"; + print " 2. orion \n"; + print " 3. gfortran compiler serial \n"; + print " 0. exit only \n"; + + printf "\nEnter selection : " ; + + $response = ; + chop($response); + } + + if ($response == 1) { + # Hera settings + system "cp config/user_build_config.hera user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load 2022.2.0 netcdf/4.7.0 \n\n\n"; + } + + elsif ($response == 2) { + # Orion settings + system "cp config/user_build_config.orion user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load intel netcdf \n\n\n"; + } + + elsif ($response == 3) { + # Mike's Mac settings + system "cp config/user_build_config.gfortran.serial user_build_config"; + } + + else {print "no selection $response\n"; last} diff --git a/tile2tile/namelist.tile2tile b/tile2tile/namelist.tile2tile new file mode 100644 index 00000000..a7a0b680 --- /dev/null +++ b/tile2tile/namelist.tile2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "vector2tile" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/restart/" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x4" + +! input files + + lndp_input_file = "/scratch2/NCEPDEV/land/data/DA/ensemble_pert/workg_T162_984x488.tileXX.nc" + +! output files + + lndp_output_file = "./output.nc" + +! land perturbation variable list + + lndp_var_list='vgf','smc' + +/ diff --git a/tile2tile/namelist_mod.f90 b/tile2tile/namelist_mod.f90 new file mode 100644 index 00000000..4fbca14b --- /dev/null +++ b/tile2tile/namelist_mod.f90 @@ -0,0 +1,86 @@ +module namelist_mod + implicit none + + integer, parameter :: max_n_var_lndp = 20 + type namelist_type + character*256 :: namelist_name = "" + character*11 :: direction = "" + character*256 :: tile_path = "" + character*256 :: tile_fstub = "" + integer :: tile_size + character*19 :: restart_date = "" + character*256 :: vector_restart_path = "" + character*256 :: tile_restart_path = "" + character*256 :: output_path = "" + character*256 :: static_filename = "" + character*3 :: lndp_layout = "" + character*256 :: lndp_input_file = "" + character*256 :: lndp_output_file = "" + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + end type namelist_type + +contains + + subroutine ReadNamelist(namelist) + + type(namelist_type) :: namelist + character*11 :: direction + character*256 :: tile_path + character*256 :: tile_fstub + integer :: tile_size + character*19 :: restart_date + character*256 :: vector_restart_path + character*256 :: tile_restart_path + character*256 :: output_path + character*256 :: static_filename + character*3 :: lndp_layout + character*256 :: lndp_input_file + character*256 :: lndp_output_file + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + integer :: k + + namelist / run_setup / direction, tile_path, tile_fstub, tile_size, restart_date, vector_restart_path, & + tile_restart_path, output_path, static_filename, lndp_layout, & + lndp_input_file, lndp_output_file, lndp_var_list, n_var_lndp + + lndp_var_list = 'XXX' + + open(30, file=namelist%namelist_name, form="formatted") + read(30, run_setup) + close(30) + + namelist%direction = direction + namelist%tile_path = tile_path + namelist%tile_fstub = tile_fstub + namelist%tile_size = tile_size + namelist%restart_date = restart_date + namelist%vector_restart_path = vector_restart_path + namelist%tile_restart_path = tile_restart_path + namelist%output_path = output_path + namelist%static_filename = static_filename + + namelist%lndp_layout = lndp_layout + namelist%lndp_input_file = lndp_input_file + namelist%lndp_output_file = lndp_output_file + + n_var_lndp= 0 + do k =1,size(lndp_var_list) + if (trim(lndp_var_list(k)) .EQ. 'XXX') then + cycle + else + n_var_lndp=n_var_lndp+1 + namelist%lndp_var_list(n_var_lndp) = lndp_var_list(k) + endif + enddo + namelist%n_var_lndp = n_var_lndp + if (n_var_lndp > max_n_var_lndp) then + print*, 'ERROR: land perturbation requested for too many parameters', & + 'increase max_n_var_lndp' + stop 10 + endif + + end subroutine ReadNamelist + +end module namelist_mod diff --git a/tile2tile/pull_request_template.md b/tile2tile/pull_request_template.md new file mode 100644 index 00000000..8eae0621 --- /dev/null +++ b/tile2tile/pull_request_template.md @@ -0,0 +1,32 @@ +PR Instructions: +1. Provide details under all headings below. +2. Assign Clara and one other person as reviewers. +3. If the PR is not ready for merging, add the "DRAFT/DO NOT MERGE" label. +4. When a PR is ready to merge, remove the "DRAFT/DO NOT MERGE" and email Clara. +5. Before requesting that the PR be merged, complete the checklist below. + +Notes on preparing PR, using git can be found in README_git + +## Describe your changes +Summarise all code changes included in PR: + +List any associated PRs in the submodules. + + +## Issue ticket number and link +List the git Issue that this PR addresses: + + +## Test output +Is this PR expected to pass the DA_IMS_test (ie., does it change the output)? + +Does it pass the DA_IMS_test? + +If changes to the test results are expected, what are these changes? Provide a link to the output directory when running the test: + +## Checklist before requesting a review +- [ ] My branch being merged is up to date with the latest develop. +- [ ] I have performed a self-review of my code by examining the differences that will be merged. +- [ ] I have not made any unnecessary code changes / changed any default behavior. +- [ ] My code passes the DA_IMS_test, or differences can be explained. + diff --git a/tile2tile/tile2tile_driver.f90 b/tile2tile/tile2tile_driver.f90 new file mode 100644 index 00000000..69a24d44 --- /dev/null +++ b/tile2tile/tile2tile_driver.f90 @@ -0,0 +1,48 @@ +program tile2tile_driver + + use namelist_mod + use tile2tile_restart_mod + implicit none + + type(namelist_type) :: namelist + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Get namelist file name from command line +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call get_command_argument(1, namelist%namelist_name) + if(namelist%namelist_name == "") then + print *, "add namelist to the command line: " + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read namelist information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadNamelist(namelist) + + print*, "conversion direction: ",namelist%direction + + if(namelist%direction /= "ufs2jedi" .and. namelist%direction /= "jedi2ufs") then + print*, "conversion direction: ",namelist%direction, " not recognized" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Decide the pathway +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + select case (namelist%direction) + + case ("ufs2jedi", "jedi2ufs") + + write(*,*) "Option: "//trim(namelist%direction) + call tile2tile_restart(namelist) + + case default + + write(*,*) "choose a valid conversion direction" + + end select +end program tile2tile_driver diff --git a/tile2tile/tile2tile_restart_mod.f90 b/tile2tile/tile2tile_restart_mod.f90 new file mode 100644 index 00000000..993260d5 --- /dev/null +++ b/tile2tile/tile2tile_restart_mod.f90 @@ -0,0 +1,893 @@ +module tile2tile_restart_mod + + use namelist_mod + use netcdf + implicit none + + type JEDI_tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type JEDI_tile_type + + type UFS_tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type UFS_tile_type + +contains + + subroutine tile2tile_restart(namelist) + type(namelist_type) :: namelist + type(JEDI_tile_type):: JEDI_tile + type(UFS_tile_type) :: UFS_tile + character*300 :: tile_filename + character*300 :: JEDI_tile_filename + character*300 :: UFS_tile_filename + character*19 :: date + integer :: yyyy,mm,dd,hh,nn,ss + integer :: itile, ix, iy, iloc + integer :: ncid, dimid, varid, status + logical :: file_exists + read(namelist%restart_date( 1: 4),'(i4.4)') yyyy + read(namelist%restart_date( 6: 7),'(i2.2)') mm + read(namelist%restart_date( 9:10),'(i2.2)') dd + read(namelist%restart_date(12:13),'(i2.2)') hh + read(namelist%restart_date(15:16),'(i2.2)') nn + read(namelist%restart_date(18:19),'(i2.2)') ss + + write(date,'(i4,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2)') & + yyyy, "-", mm, "-", dd, "_", hh, "-", nn, "-", ss + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(JEDI_tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(JEDI_tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(JEDI_tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%soil_moisture_total (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(JEDI_tile%soil_moisture_liquid (namelist%tile_size,namelist%tile_size,4,6)) + allocate(JEDI_tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + + allocate(UFS_tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(UFS_tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(UFS_tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%soil_moisture_total(namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(UFS_tile%soil_moisture_liquid(namelist%tile_size,namelist%tile_size,4,6)) + allocate(UFS_tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a5,i1,a3)') ".tile", itile, ".nc" + + tile_filename = trim(namelist%tile_path)//trim(namelist%tile_fstub)//trim(adjustl(tile_filename)) + inquire(file=trim(tile_filename), exist=file_exists) + + print*, "Reading tile file: ", trim(tile_filename) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist1" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(trim(tile_filename), NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , JEDI_tile%land_frac(:,:,itile)) + status = nf90_get_var(ncid, varid , UFS_tile%land_frac(:,:,itile)) + + status = nf90_close(ncid) + + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Direction of transfer branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(namelist%direction == "ufs2jedi") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read UFS NoahMP restart file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call UFS_ReadTileRestart(namelist, date, UFS_tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! explicitly initialize to 0. + JEDI_tile%slmsk=0. + + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(UFS_tile%land_frac(ix,iy,itile) > 0.0) then + JEDI_tile%swe(ix,iy,itile) = UFS_tile%swe(ix,iy,itile) + JEDI_tile%vegetation_type(ix,iy,itile) = UFS_tile%vegetation_type(ix,iy,itile) + JEDI_tile%snow_depth(ix,iy,itile) = UFS_tile%snow_depth(ix,iy,itile) + JEDI_tile%active_snow_layers(ix,iy,itile) = UFS_tile%active_snow_layers(ix,iy,itile) + JEDI_tile%swe_previous(ix,iy,itile) = UFS_tile%swe_previous(ix,iy,itile) + JEDI_tile%snow_soil_interface(ix,iy,:,itile) = UFS_tile%snow_soil_interface(ix,iy,:,itile) + JEDI_tile%temperature_snow(ix,iy,:,itile) = UFS_tile%temperature_snow(ix,iy,:,itile) + JEDI_tile%snow_ice_layer(ix,iy,:,itile) = UFS_tile%snow_ice_layer(ix,iy,:,itile) + JEDI_tile%snow_liq_layer(ix,iy,:,itile) = UFS_tile%snow_liq_layer(ix,iy,:,itile) + JEDI_tile%temperature_soil(ix,iy,:,itile) = UFS_tile%temperature_soil(ix,iy,:,itile) + JEDI_tile%soil_moisture_total(ix,iy,:,itile) = UFS_tile%soil_moisture_total(ix,iy,:,itile) + JEDI_tile%slmsk(ix,iy,itile) = 1. + JEDI_tile%soil_moisture_liquid(ix,iy,:,itile)= UFS_tile%soil_moisture_liquid(ix,iy,:,itile) + JEDI_tile%temperature_ground(ix,iy,itile) = UFS_tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write JEDI tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call JEDI_WriteTileRestart(namelist, date, JEDI_tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! JEDI2UFS branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + elseif(namelist%direction == "jedi2ufs") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!! Read JEDI tile restart files +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! + call JEDI_ReadTileRestart(namelist, date, JEDI_tile) +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +! explicitly initialize to 0. + UFS_tile%slmsk=0. + + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(JEDI_tile%land_frac(ix,iy,itile) > 0.0) then + UFS_tile%swe(ix,iy,itile) = JEDI_tile%swe(ix,iy,itile) + UFS_tile%vegetation_type(ix,iy,itile) = JEDI_tile%vegetation_type(ix,iy,itile) + UFS_tile%snow_depth(ix,iy,itile) = JEDI_tile%snow_depth(ix,iy,itile) + UFS_tile%active_snow_layers(ix,iy,itile) = JEDI_tile%active_snow_layers(ix,iy,itile) + UFS_tile%swe_previous(ix,iy,itile) = JEDI_tile%swe_previous(ix,iy,itile) + UFS_tile%snow_soil_interface(ix,iy,:,itile) = JEDI_tile%snow_soil_interface(ix,iy,:,itile) + UFS_tile%temperature_snow(ix,iy,:,itile) = JEDI_tile%temperature_snow(ix,iy,:,itile) + UFS_tile%snow_ice_layer(ix,iy,:,itile) = JEDI_tile%snow_ice_layer(ix,iy,:,itile) + UFS_tile%snow_liq_layer(ix,iy,:,itile) = JEDI_tile%snow_liq_layer(ix,iy,:,itile) + UFS_tile%temperature_soil(ix,iy,:,itile) = JEDI_tile%temperature_soil(ix,iy,:,itile) + UFS_tile%soil_moisture_total(ix,iy,:,itile) = JEDI_tile%soil_moisture_total(ix,iy,:,itile) + UFS_tile%slmsk(ix,iy,itile) = 1. + UFS_tile%soil_moisture_liquid(ix,iy,:,itile)= JEDI_tile%soil_moisture_liquid(ix,iy,:,itile) + UFS_tile%temperature_ground(ix,iy,itile) = JEDI_tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write UFS tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! + call UFS_WriteTileRestart(namelist, date, UFS_tile) +! + end if ! "ufs2jedi" or "jedi2ufs" branch + + end subroutine tile2tile_restart + + subroutine UFS_ReadTileRestart(namelist, date, UFS_tile) + + use netcdf + + type(namelist_type) :: namelist + type(UFS_tile_type) :: UFS_tile + character*19 :: date + character*256 :: UFS_tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(UFS_tile_filename,'(a17,a19,a5,i1,a3)') & + "ufs_land_restart.", date, ".tile",itile,".nc" +! date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + UFS_tile_filename = trim(namelist%tile_restart_path)//trim(UFS_tile_filename) + + print*, "Reading UFS tile file: ", trim(UFS_tile_filename) + + inquire(file=UFS_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(UFS_tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the UFS NoahMP tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(UFS_tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading restart file + + status = nf90_inq_varid(ncid, "weasd", varid) + if (status /= nf90_noerr) then + print *, 'weasd variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_soil(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vegtype", varid) + if (status /= nf90_noerr) then + print *, 'vegtype variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , UFS_tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine UFS_ReadTileRestart + + + subroutine JEDI_WriteTileRestart(namelist, date, JEDI_tile) + + use netcdf + + type(namelist_type) :: namelist + type(JEDI_tile_type):: JEDI_tile + character*19 :: date + character*256 :: JEDI_tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow, dim_id_snso, dim_id_time + + do itile = 1, 6 + + write(JEDI_tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + JEDI_tile_filename = trim(namelist%output_path)//trim(JEDI_tile_filename) + + print*, "Writing tile file: ", trim(JEDI_tile_filename) + + status = nf90_create(JEDI_tile_filename, NF90_CLOBBER, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid, "xaxis_1" , namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "yaxis_1" , namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_2" , 4 , dim_id_soil) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_3" , 3 , dim_id_snow) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_4" , 7 , dim_id_snso) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "Time" , NF90_UNLIMITED , dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! define dimension variables (for JEDI) + + status = nf90_def_var(ncid, "Time", NF90_DOUBLE, & + (/dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "xaxis_1", NF90_DOUBLE, & + (/dim_id_xdim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "yaxis_1", NF90_DOUBLE, & + (/dim_id_ydim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_2", NF90_DOUBLE, & + (/dim_id_soil/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_3", NF90_DOUBLE, & + (/dim_id_snow/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_4", NF90_DOUBLE, & + (/dim_id_snso/), varid) + if (status /= nf90_noerr) call handle_err(status) + + +! Define variables in the file. + + status = nf90_def_var(ncid, "sheleg", NF90_DOUBLE, & ! note: this is weasd in vector file. + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snwdph", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snowxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "sneqvoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zsnsoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snso,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tsnoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snicexy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snliqxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "stc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "smc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slmsk", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "vtype", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tgxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_enddef(ncid) + + +! fill dimension variables + + status = nf90_inq_varid(ncid, "Time", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/1/) ) + + status = nf90_inq_varid(ncid, "xaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "yaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "zaxis_2", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 4)/) ) + + status = nf90_inq_varid(ncid, "zaxis_3", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 3)/) ) + + status = nf90_inq_varid(ncid, "zaxis_4", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 7)/) ) + +! Start writing restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_soil(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + +! include in output, so can be used to id which tile grid cells are being simulated + status = nf90_inq_varid(ncid, "slmsk", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%slmsk(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + +! include for JEDI QC of SMAP obs + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , JEDI_tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine JEDI_WriteTileRestart + + + subroutine JEDI_ReadTileRestart(namelist, date, JEDI_tile) + + use netcdf + + type(namelist_type) :: namelist + type(JEDI_tile_type) :: JEDI_tile + character*19 :: date + character*256 :: JEDI_tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + + write(JEDI_tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13),"0000.sfc_data.tile",itile,".nc" + + JEDI_tile_filename = trim(namelist%output_path)//trim(JEDI_tile_filename) + + print*, "Reading JEDI tile file: ", trim(JEDI_tile_filename) + + inquire(file=JEDI_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(JEDI_tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the JEDI tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(JEDI_tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading updated file + + status = nf90_inq_varid(ncid, "sheleg", varid) + if (status /= nf90_noerr) then + print *, 'sheleg variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%active_snow_layers(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid ,JEDI_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_snow(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_soil(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%soil_moisture_total(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid ,JEDI_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%temperature_ground(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + if (status /= nf90_noerr) then + print *, 'vegtype variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , JEDI_tile%vegetation_type(:,:,itile) ,& + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine JEDI_ReadTileRestart + + subroutine UFS_WriteTileRestart(namelist, date, UFS_tile) + + use netcdf + + type(namelist_type) :: namelist + type(UFS_tile_type):: UFS_tile + character*19 :: date + character*256 :: UFS_tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow,dim_id_snso, dim_id_time + logical :: file_exists + + do itile = 1, 6 + + write(UFS_tile_filename,'(a17,a19,a5,i1,a3)') & + "ufs_land_restart.", date, ".tile",itile,".nc" + + UFS_tile_filename = trim(namelist%output_path)//trim(UFS_tile_filename) + + inquire(file=UFS_tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(UFS_tile_filename), " does not exist5" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Writing tile file: ", trim(UFS_tile_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write the tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(UFS_tile_filename, NF90_WRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "weasd", varid) + status = nf90_put_var(ncid, varid , UFS_tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%active_snow_layers(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid ,UFS_tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_snow(:,:,:,itile), & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_ice_layer(:,:,:,itile) ,& + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%snow_liq_layer(:,:,:,itile) ,& + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_soil(:,:,:,itile), & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4,1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid ,UFS_tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4,1/)) + + status = nf90_inq_varid(ncid, "vegtype", varid) + status = nf90_put_var(ncid, varid , UFS_tile%vegetation_type(:,:,itile) ,& + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , UFS_tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , UFS_tile%temperature_ground(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + end subroutine UFS_WriteTileRestart + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module tile2tile_restart_mod diff --git a/tile2tile/user_build_config b/tile2tile/user_build_config new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/tile2tile/user_build_config @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/vector2tile b/vector2tile deleted file mode 160000 index 2fb5c1b4..00000000 --- a/vector2tile +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2fb5c1b493b3c03809e081f6fe8e8e76a40a100d diff --git a/vector2tile/.gitignore b/vector2tile/.gitignore new file mode 100644 index 00000000..18de9433 --- /dev/null +++ b/vector2tile/.gitignore @@ -0,0 +1,4 @@ +vector2tile_converter.exe +vector2tile_converter.o +*.mod +*.o diff --git a/vector2tile/CMakeLists.txt b/vector2tile/CMakeLists.txt new file mode 100644 index 00000000..81ac1f91 --- /dev/null +++ b/vector2tile/CMakeLists.txt @@ -0,0 +1,76 @@ +# (C) Copyright 2022 . +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +cmake_minimum_required( VERSION 3.12) +project( vector2tile VERSION 2022.10 LANGUAGES Fortran ) + +# ecbuild integration +find_package(ecbuild 3.3.2 REQUIRED) +include( ecbuild_system NO_POLICY_SCOPE ) +ecbuild_declare_project() + +list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +include( vector2tile_compiler_flags ) + +################################################################################ +# Dependencies +################################################################################ + +find_package(OpenMP COMPONENTS C Fortran) +find_package(MPI REQUIRED COMPONENTS C Fortran) +find_package(NetCDF REQUIRED COMPONENTS Fortran ) + +################################################################################ +# Sources +################################################################################ + +set( VECTOR2TILE_LINKER_LANGUAGE Fortran ) + +list ( APPEND src_files +./namelist_mod.f90 +./vector2tile_perturbation_mod.f90 +./vector2tile_restart_mod.f90 +) + +ecbuild_add_library( TARGET vector2tile + SOURCES ${src_files} + INSTALL_HEADERS LISTED + LINKER_LANGUAGE ${VECTOR2TILE_LINKER_LANGUAGE} + ) + + +target_link_libraries(vector2tile PUBLIC NetCDF::NetCDF_Fortran) +target_link_libraries(vector2tile PUBLIC MPI::MPI_Fortran) +target_link_libraries(vector2tile PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_Fortran) + +# Fortran module output directory for build and install interfaces +set(MODULE_DIR module/${PROJECT_NAME}/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}) +set_target_properties(${PROJECT_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}) +install(DIRECTORY ${CMAKE_BINARY_DIR}/${MODULE_DIR}/ DESTINATION ${MODULE_DIR}) +target_include_directories(${PROJECT_NAME} INTERFACE + $ + $) + +set( VECTOR2TILE_INCLUDE_DIRS ${CMAKE_Fortran_MODULE_DIRECTORY} ) +set( VECTOR2TILE_LIBRARIES vector2tile ) + +# vector2tile_converter.exe executable +#------------------------------------------------------------------------------- +set ( exe_files vector2tile_driver.f90 ) + +ecbuild_add_executable( TARGET vector2tile_converter.exe + SOURCES ${exe_files} + LIBS vector2tile + LINKER_LANGUAGE ${VECTOR2TILE_LINKER_LANGUAGE} + ) + + +################################################################################ +# Finalise configuration +################################################################################ + +ecbuild_install_project( NAME vector2tile ) +ecbuild_print_summary() diff --git a/vector2tile/Makefile b/vector2tile/Makefile new file mode 100644 index 00000000..45308c7e --- /dev/null +++ b/vector2tile/Makefile @@ -0,0 +1,24 @@ +# Makefile +# +.SUFFIXES: +.SUFFIXES: .o .f90 + +include ./user_build_config + +OBJS = namelist_mod.o vector2tile_restart_mod.o vector2tile_perturbation_mod.o vector2tile_driver.o + +all: vector2tile_converter.exe + +.f90.o: + $(COMPILERF90) -c $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(*).f90 + +vector2tile_converter.exe: $(OBJS) + $(COMPILERF90) -o $(@) $(F90FLAGS) $(FREESOURCE) $(NETCDFMOD) $(OBJS) $(NETCDFLIB) + +clean: + rm -f *.o *.mod *.exe + + +# +# Dependencies: +# diff --git a/vector2tile/README.md b/vector2tile/README.md new file mode 100644 index 00000000..69c46d60 --- /dev/null +++ b/vector2tile/README.md @@ -0,0 +1,30 @@ +Code to map between vector format used by the Noah-MP offline driver, and the tile format used by the UFS atmospheric model. Currently used to prepare input tile files for JEDI. These files include only those fields required by JEDI, rather than the full restart. Can also be used to map stochastic_physics output to the tile or vector. Stochastic physics output files depend on the layout, with 1x1 layout giving one file per tile. + +Mike Barlage, Clara Draper. Dec 2021. + +To compile on hera: + +>configure + + choose hera + + load the modules indicated + +>make + +To run: + +>vector2tile_converter.exe namelist.vector2tile + +the namelist defines the conversion direction and the paths of the files + +Details: + +the vector2tile pathway assumes that the vector file exists in the vector_restart_path directory and overwrites/creates tile files in the output_path + +the tile2vector pathway is a little tricky, it assumes the tile files exist in tile_restart_path and overwrites only the snow variables in the vector file in the output_path. If the vector file does not exist in output_path the process will fail. + +the overall assumption here is that we will have a full model vector restart file, then convert the vector to tiles for only snow variables, then convert the updated snow variables back to the full vector restart file + +For the lndp2vector or lndp2tile option, a new file will be created with the pertbations and lat lon only. + diff --git a/vector2tile/cmake/compiler_flags_GNU_CXX.cmake b/vector2tile/cmake/compiler_flags_GNU_CXX.cmake new file mode 100755 index 00000000..dbe13c02 --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_CXX.cmake @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +if( HAVE_OMP ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fopenmp") +else( ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations -fno-openmp") +endif( ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_DEBUG "-O0 -g" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_CXX_FLAGS_BIT "-O2" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_CXX_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake new file mode 100755 index 00000000..c0b618cc --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake @@ -0,0 +1,43 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test new file mode 100755 index 00000000..95f6209b --- /dev/null +++ b/vector2tile/cmake/compiler_flags_GNU_Fortran.cmake_test @@ -0,0 +1,45 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8 -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace") +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -fbacktrace -ffree-line-length-none -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -ffree-form") + +#################################################################### +# RELEASE FLAGS +#################################################################### + +#jkim set( CMAKE_Fortran_FLAGS_RELEASE "-O3" ) +set( CMAKE_Fortran_FLAGS_RELEASE " " ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -fbounds-check" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# -fstack-arrays : Allocate automatic arrays on the stack (needs large stacksize!!!) +# -funroll-all-loops : Unroll all loops +# -fcheck=bounds : Bounds checking + diff --git a/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake b/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake new file mode 100755 index 00000000..7eef4052 --- /dev/null +++ b/vector2tile/cmake/compiler_flags_Intel_Fortran.cmake @@ -0,0 +1,41 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#################################################################### +# FLAGS COMMON TO ALL BUILD TYPES +#################################################################### + +set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -convert big_endian -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -i4 -r8 -nowarn -sox -traceback -msse2" ) + +#################################################################### +# RELEASE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_RELEASE "-O3 -debug minimal -fp-model source" ) + +#################################################################### +# DEBUG FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_DEBUG "-g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fpe0 -ftrapuv" ) + +#################################################################### +# BIT REPRODUCIBLE FLAGS +#################################################################### + +set( CMAKE_Fortran_FLAGS_BIT "-O2 -debug minimal -fp-model source" ) + +#################################################################### +# LINK FLAGS +#################################################################### + +set( CMAKE_Fortran_LINK_FLAGS "" ) + +#################################################################### + +# Meaning of flags +# ---------------- +# todo + diff --git a/vector2tile/cmake/vector2tile_compiler_flags.cmake b/vector2tile/cmake/vector2tile_compiler_flags.cmake new file mode 100755 index 00000000..8aebfaec --- /dev/null +++ b/vector2tile/cmake/vector2tile_compiler_flags.cmake @@ -0,0 +1,23 @@ +# (C) Copyright 2018 UCAR. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +#add_definitions ( -Duse_libMPI -Duse_netCDF -DSPMD -DNXGLOB=4) +add_definitions ( -Duse_libMPI -Duse_netCDF -Dncdf -Dgather_scatter_barrier) + +####################################################################################### +# Fortran +####################################################################################### + +if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) + include( compiler_flags_GNU_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) + include( compiler_flags_Intel_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "XL" ) + include( compiler_flags_XL_Fortran ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" ) + include( compiler_flags_Cray_Fortran ) +else() + message( STATUS "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options") +endif() diff --git a/vector2tile/config/user_build_config.gfortran.serial b/vector2tile/config/user_build_config.gfortran.serial new file mode 100644 index 00000000..a7191d5c --- /dev/null +++ b/vector2tile/config/user_build_config.gfortran.serial @@ -0,0 +1,11 @@ + +#=============================================================================== +# Placeholder options for Mac gfortran +#=============================================================================== + + COMPILERF90 = /opt/local/bin/gfortran-mp-11 + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/opt/local/include + NETCDFLIB = -L/opt/local/lib -lnetcdf -lnetcdff + diff --git a/vector2tile/config/user_build_config.hera b/vector2tile/config/user_build_config.hera new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/vector2tile/config/user_build_config.hera @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/vector2tile/configure b/vector2tile/configure new file mode 100755 index 00000000..24aad730 --- /dev/null +++ b/vector2tile/configure @@ -0,0 +1,39 @@ +#!/usr/bin/perl + + if($#ARGV == 0) { + $response = shift(@ARGV) ; + print("Configure soil test environment: $response \n"); + }else { + print "Please select from following predefined environments: \n\n"; + + print " 1. hera \n"; + print " 2. orion \n"; + print " 3. gfortran compiler serial \n"; + print " 0. exit only \n"; + + printf "\nEnter selection : " ; + + $response = ; + chop($response); + } + + if ($response == 1) { + # Hera settings + system "cp config/user_build_config.hera user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load 2022.2.0 netcdf/4.7.0 \n\n\n"; + } + + elsif ($response == 2) { + # Orion settings + system "cp config/user_build_config.orion user_build_config"; + print "\n load necessary modules:\n"; + print "\n module load intel netcdf \n\n\n"; + } + + elsif ($response == 3) { + # Mike's Mac settings + system "cp config/user_build_config.gfortran.serial user_build_config"; + } + + else {print "no selection $response\n"; last} diff --git a/vector2tile/namelist.lndp2vector b/vector2tile/namelist.lndp2vector new file mode 100644 index 00000000..92ade568 --- /dev/null +++ b/vector2tile/namelist.lndp2vector @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "lndp2vector" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "junk" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "junk" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "./" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x1, 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x1" + +! input files + + lndp_input_file = "./stochy_out1/workg_T162_984x488.tile01.nc" + +! output files + + lndp_output_file = "./output_ens001.nc" + +! land perturbation variable list + + lndp_var_list='vgf' + +/ diff --git a/vector2tile/namelist.vector2tile b/vector2tile/namelist.vector2tile new file mode 100644 index 00000000..a7a0b680 --- /dev/null +++ b/vector2tile/namelist.vector2tile @@ -0,0 +1,52 @@ +&run_setup + +!------------------- common ------------------- +! Direction of conversion: either "vector2tile" or "tile2vector" for restart file +! "lndp2tile" or "lndp2vector" for perturbation + + direction = "vector2tile" + +! FV3 resolution and path to oro files for restart/perturbation conversion + + tile_size = 96 + tile_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/tile_files/C96.mx100_frac/" + tile_fstub = "oro_C96.mx100" + +!------------------- only restart conversion ------------------- +! Time stamp for conversion for restart conversion + + restart_date = "2019-09-30 23:00:00" + +! Path for static file + static_filename="/scratch1/NCEPDEV/stmp2/Michael.Barlage/forcing/C96/static/ufs-land_C96_static_fields.nc" + +! Location of vector restart file (vector2tile direction) + + vector_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/restart/" + +! Location of tile restart files (tile2vector direction) + + tile_restart_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +! Path for converted files; if same as tile/vector path, files may be overwritten + + output_path = "/scratch1/NCEPDEV/stmp2/Michael.Barlage/models/vector/v2t_data/workshop/" + +!------------------- only perturbation mapping ------------------- +! layout, options: 1x4, 4x1, 2x2, an input settings for generating the perturbation file + + lndp_layout = "1x4" + +! input files + + lndp_input_file = "/scratch2/NCEPDEV/land/data/DA/ensemble_pert/workg_T162_984x488.tileXX.nc" + +! output files + + lndp_output_file = "./output.nc" + +! land perturbation variable list + + lndp_var_list='vgf','smc' + +/ diff --git a/vector2tile/namelist_mod.f90 b/vector2tile/namelist_mod.f90 new file mode 100644 index 00000000..4fbca14b --- /dev/null +++ b/vector2tile/namelist_mod.f90 @@ -0,0 +1,86 @@ +module namelist_mod + implicit none + + integer, parameter :: max_n_var_lndp = 20 + type namelist_type + character*256 :: namelist_name = "" + character*11 :: direction = "" + character*256 :: tile_path = "" + character*256 :: tile_fstub = "" + integer :: tile_size + character*19 :: restart_date = "" + character*256 :: vector_restart_path = "" + character*256 :: tile_restart_path = "" + character*256 :: output_path = "" + character*256 :: static_filename = "" + character*3 :: lndp_layout = "" + character*256 :: lndp_input_file = "" + character*256 :: lndp_output_file = "" + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + end type namelist_type + +contains + + subroutine ReadNamelist(namelist) + + type(namelist_type) :: namelist + character*11 :: direction + character*256 :: tile_path + character*256 :: tile_fstub + integer :: tile_size + character*19 :: restart_date + character*256 :: vector_restart_path + character*256 :: tile_restart_path + character*256 :: output_path + character*256 :: static_filename + character*3 :: lndp_layout + character*256 :: lndp_input_file + character*256 :: lndp_output_file + character(len=128) :: lndp_var_list(max_n_var_lndp) + integer :: n_var_lndp + integer :: k + + namelist / run_setup / direction, tile_path, tile_fstub, tile_size, restart_date, vector_restart_path, & + tile_restart_path, output_path, static_filename, lndp_layout, & + lndp_input_file, lndp_output_file, lndp_var_list, n_var_lndp + + lndp_var_list = 'XXX' + + open(30, file=namelist%namelist_name, form="formatted") + read(30, run_setup) + close(30) + + namelist%direction = direction + namelist%tile_path = tile_path + namelist%tile_fstub = tile_fstub + namelist%tile_size = tile_size + namelist%restart_date = restart_date + namelist%vector_restart_path = vector_restart_path + namelist%tile_restart_path = tile_restart_path + namelist%output_path = output_path + namelist%static_filename = static_filename + + namelist%lndp_layout = lndp_layout + namelist%lndp_input_file = lndp_input_file + namelist%lndp_output_file = lndp_output_file + + n_var_lndp= 0 + do k =1,size(lndp_var_list) + if (trim(lndp_var_list(k)) .EQ. 'XXX') then + cycle + else + n_var_lndp=n_var_lndp+1 + namelist%lndp_var_list(n_var_lndp) = lndp_var_list(k) + endif + enddo + namelist%n_var_lndp = n_var_lndp + if (n_var_lndp > max_n_var_lndp) then + print*, 'ERROR: land perturbation requested for too many parameters', & + 'increase max_n_var_lndp' + stop 10 + endif + + end subroutine ReadNamelist + +end module namelist_mod diff --git a/vector2tile/pull_request_template.md b/vector2tile/pull_request_template.md new file mode 100644 index 00000000..8eae0621 --- /dev/null +++ b/vector2tile/pull_request_template.md @@ -0,0 +1,32 @@ +PR Instructions: +1. Provide details under all headings below. +2. Assign Clara and one other person as reviewers. +3. If the PR is not ready for merging, add the "DRAFT/DO NOT MERGE" label. +4. When a PR is ready to merge, remove the "DRAFT/DO NOT MERGE" and email Clara. +5. Before requesting that the PR be merged, complete the checklist below. + +Notes on preparing PR, using git can be found in README_git + +## Describe your changes +Summarise all code changes included in PR: + +List any associated PRs in the submodules. + + +## Issue ticket number and link +List the git Issue that this PR addresses: + + +## Test output +Is this PR expected to pass the DA_IMS_test (ie., does it change the output)? + +Does it pass the DA_IMS_test? + +If changes to the test results are expected, what are these changes? Provide a link to the output directory when running the test: + +## Checklist before requesting a review +- [ ] My branch being merged is up to date with the latest develop. +- [ ] I have performed a self-review of my code by examining the differences that will be merged. +- [ ] I have not made any unnecessary code changes / changed any default behavior. +- [ ] My code passes the DA_IMS_test, or differences can be explained. + diff --git a/vector2tile/user_build_config b/vector2tile/user_build_config new file mode 100644 index 00000000..6ed29784 --- /dev/null +++ b/vector2tile/user_build_config @@ -0,0 +1,10 @@ + +#=============================================================================== +# Placeholder options for hera +#=============================================================================== + + COMPILERF90 = ifort + FREESOURCE = + F90FLAGS = + NETCDFMOD = -I/apps/netcdf/4.7.0/intel/18.0.5.274/include + NETCDFLIB = -L/apps/netcdf/4.7.0/intel/18.0.5.274/lib -lnetcdf -lnetcdff diff --git a/vector2tile/vector2tile_driver.f90 b/vector2tile/vector2tile_driver.f90 new file mode 100644 index 00000000..988d49d3 --- /dev/null +++ b/vector2tile/vector2tile_driver.f90 @@ -0,0 +1,55 @@ +program vector2tile_driver + + use namelist_mod + use vector2tile_restart_mod + use vector2tile_perturbation_mod + implicit none + + type(namelist_type) :: namelist + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Get namelist file name from command line +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call get_command_argument(1, namelist%namelist_name) + if(namelist%namelist_name == "") then + print *, "add namelist to the command line: " + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read namelist information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadNamelist(namelist) + + print*, "conversion direction: ",namelist%direction + + if(namelist%direction /= "tile2vector" .and. namelist%direction /= "vector2tile" .and. & + namelist%direction /= "lndp2vector" .and. namelist%direction /= "lndp2tile") then + print*, "conversion direction: ",namelist%direction, " not recognized" + stop 10 + end if + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Decide the pathway +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + select case (namelist%direction) + + case ("tile2vector", "vector2tile") + + write(*,*) "Option: "//trim(namelist%direction) + call vector2tile_restart(namelist) + + case ("lndp2vector", "lndp2tile") + + write(*,*) "Option: "//trim(namelist%direction) + call mapping_perturbation(namelist) + + case default + + write(*,*) "choose a valid conversion direction" + + end select +end program vector2tile_driver diff --git a/vector2tile/vector2tile_perturbation_mod.f90 b/vector2tile/vector2tile_perturbation_mod.f90 new file mode 100644 index 00000000..1b0f977a --- /dev/null +++ b/vector2tile/vector2tile_perturbation_mod.f90 @@ -0,0 +1,347 @@ +module vector2tile_perturbation_mod + + use netcdf + use namelist_mod + implicit none + +contains + + subroutine mapping_perturbation(namelist) + type(namelist_type) :: namelist + real, allocatable :: var_vector(:), lon_vector(:), lat_vector(:) + real, allocatable :: var_tile(:,:,:), lon_tile(:,:,:), lat_tile(:,:,:) + real, allocatable :: land_frac_tile(:,:,:) + character*256 :: vector_filename + character*256 :: tile_filename + character*256 :: input_filename + character*256 :: output_filename + character*2 :: tile1, tile2 + real, allocatable :: tmp2d(:,:) + integer :: filename_length + integer :: vector_length = 0 + integer :: layout_x, layout_y, nx, ny + integer :: itile, ix, iy, iloc, ivar + integer :: i, j, m, n, i1, i2, j1, j2, t2 + integer :: ncid, dimid, varid, status + integer :: dim_id_xdim, dim_id_ydim, dim_id_time + integer :: ncid_landp, ncid_vec, ncid_tile(6) + logical :: file_exists + + if(trim(namelist%lndp_layout) == '1x4') then + layout_x = 1 + layout_y = 4 + else if(trim(namelist%lndp_layout) == '2x2') then + layout_x = 2 + layout_y = 2 + else if(trim(namelist%lndp_layout) == '4x1') then + layout_x = 4 + layout_y = 1 + else if(trim(namelist%lndp_layout) == '1x1') then + layout_x = 1 + layout_y = 1 + else + print*, "layout: ",namelist%lndp_layout, " not recognized" + stop 10 + endif + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + allocate(var_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(lon_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(lat_tile( namelist%tile_size,namelist%tile_size,6)) + allocate(land_frac_tile(namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + + if(namelist%tile_size < 100) then + write(tile_filename,'(a5,i2,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + elseif(namelist%tile_size < 1000) then + write(tile_filename,'(a5,i3,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + elseif(namelist%tile_size < 10000) then + write(tile_filename,'(a5,i4,a11,i1,a3)') "oro_C", namelist%tile_size, ".mx100.tile", itile, ".nc" + else + print *, "unknown tile size" + stop 10 + end if + + tile_filename = trim(namelist%tile_path)//trim(tile_filename) + + inquire(file=tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist" + print*, "In routine mapping_perturbation: check paths and file name" + stop 10 + end if + + status = nf90_open(tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , land_frac_tile(:,:,itile)) + + status = nf90_close(ncid) + + vector_length = vector_length + count(land_frac_tile(:,:,itile) > 0) + + end do + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate vector variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(var_vector(vector_length)) + allocate(lon_vector(vector_length)) + allocate(lat_vector(vector_length)) + + nx = namelist%tile_size/layout_x + ny = namelist%tile_size/layout_y + + if(namelist%n_var_lndp > 0) then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the output file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if(trim(namelist%direction) == "lndp2vector") then + + output_filename = namelist%lndp_output_file + status = nf90_create(output_filename, NF90_CLOBBER, ncid_vec) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid_vec, "location", vector_length, dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_vec, "Time", NF90_UNLIMITED, dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! Define lat and lon + status = nf90_def_var(ncid_vec, 'lon_vec', & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_var(ncid_vec, 'lat_vec', & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + +! Define variables in the file. + do ivar = 1, namelist%n_var_lndp + status = nf90_def_var(ncid_vec, namelist%lndp_var_list(ivar), & + NF90_FLOAT, (/dim_id_xdim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + enddo + status = nf90_enddef(ncid_vec) + + else if(trim(namelist%direction) == "lndp2tile" ) then + + filename_length = len_trim(namelist%lndp_output_file) + do itile = 1, 6 + write(tile1,fmt='(I2.2)') itile + output_filename = trim(namelist%lndp_output_file(1:filename_length-5))//trim(tile1)//'.nc' + status = nf90_create(output_filename, NF90_CLOBBER, ncid_tile(itile)) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid_tile(itile), "xaxis_1", namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_tile(itile), "yaxis_1", namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid_tile(itile), "Time", NF90_UNLIMITED, dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! Define lat and lon in the file + status = nf90_def_var(ncid_tile(itile), 'lon_tile', & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_var(ncid_tile(itile), 'lat_tile', & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + +! Define variables in the file. + do ivar = 1, namelist%n_var_lndp + status = nf90_def_var(ncid_tile(itile), namelist%lndp_var_list(ivar), & + NF90_FLOAT, (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + enddo + status = nf90_enddef(ncid_tile(itile)) + enddo + endif + + allocate(tmp2d(nx,ny)) + + do ivar = 1, namelist%n_var_lndp + t2 = 1 + filename_length = len_trim(namelist%lndp_input_file) + do itile = 1, 6 + i1=1 + i2=i1+nx-1 + j1=1 + j2=j1+ny-1 + do j=1,layout_y + do i=1,layout_x + write(tile2,fmt='(I2.2)') t2 + if(t2 > 1) then + i1=i1+nx + i2=i2+nx + if (i2 .GT. namelist%tile_size) then + i1=1 + i2=i1+nx-1 + endif + endif + input_filename = trim(namelist%lndp_input_file(1:filename_length-5))//trim(tile2)//'.nc' + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Read the perturbation pattern + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(input_filename, NF90_NOWRITE, ncid_landp) + status = nf90_inq_varid(ncid_landp, namelist%lndp_var_list(ivar), varid) + if (status /= nf90_noerr) then + print *, trim(namelist%lndp_var_list(ivar))//' variable missing from perturbation file' + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + var_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + ! get the lat and lon + if ( ivar == 1 ) then + status = nf90_inq_varid(ncid_landp, 'grid_lon', varid) + if (status /= nf90_noerr) then + print *, trim('grid lon variable missing from perturbation file') + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + lon_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + ! get the lat and lat + status = nf90_inq_varid(ncid_landp, 'grid_lat', varid) + if (status /= nf90_noerr) then + print *, trim('grid lat variable missing from perturbation file') + call handle_err(status) + endif + + status = nf90_get_var(ncid_landp, varid, tmp2d, start = (/1,1,1/), count = (/nx, ny, 1/)) + + do m = i1, i2 + do n = j1, j2 + lat_tile(m,n,itile) = tmp2d(m-i1+1,n-j1+1) + enddo + enddo + + endif + + t2 = t2+1 + + enddo + + j1=j1+ny + j2=j2+ny + + if (j2 .GT. namelist%tile_size) then + j1=1 + j2=j1+ny-1 + endif + + enddo + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Write the perturbation pattern for the tile files + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2tile") then + status = nf90_inq_varid(ncid_tile(itile), namelist%lndp_var_list(ivar), varid) + status = nf90_put_var(ncid_tile(itile), varid , var_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + if (ivar==1) then + status = nf90_inq_varid(ncid_tile(itile), 'lon_grid', varid) + status = nf90_put_var(ncid_tile(itile), varid , lon_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + status = nf90_inq_varid(ncid_tile(itile), 'lat_grid', varid) + status = nf90_put_var(ncid_tile(itile), varid , lat_tile(:,:,itile), & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + endif + endif + + enddo ! for each tile + + + iloc = 0 + do itile = 1, 6 + do j = 1, namelist%tile_size + do i = 1, namelist%tile_size + if(land_frac_tile(i,j,itile) > 0.0) then + iloc = iloc + 1 + var_vector(iloc) = var_tile(i,j,itile) + if (ivar==1) then + lon_vector(iloc) = lon_tile(i,j,itile) + lat_vector(iloc) = lat_tile(i,j,itile) + endif + endif + enddo + enddo + enddo ! for each tile + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Write the perturbation pattern for the vector file + !!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2vector") then + status = nf90_inq_varid(ncid_vec, namelist%lndp_var_list(ivar), varid) + status = nf90_put_var(ncid_vec, varid , var_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + + if (ivar == 1) then + status = nf90_inq_varid(ncid_vec, 'lon_vec', varid) + status = nf90_put_var(ncid_vec, varid , lon_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + status = nf90_inq_varid(ncid_vec, 'lat_vec', varid) + status = nf90_put_var(ncid_vec, varid , lat_vector(:), & + start = (/1,1/), count = (/vector_length, 1/)) + endif + endif + + enddo ! for each variable + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Close the netcdf file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(trim(namelist%direction) == "lndp2vector") then + status = nf90_close(ncid_vec) + else if(trim(namelist%direction) == "lndp2tile") then + do itile = 1, 6 + status = nf90_close(ncid_tile(itile)) + enddo + endif + endif + end subroutine mapping_perturbation + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module vector2tile_perturbation_mod diff --git a/vector2tile/vector2tile_restart_mod.f90 b/vector2tile/vector2tile_restart_mod.f90 new file mode 100644 index 00000000..1679294c --- /dev/null +++ b/vector2tile/vector2tile_restart_mod.f90 @@ -0,0 +1,929 @@ +module vector2tile_restart_mod + + use namelist_mod + use netcdf + implicit none + + type vector_type + double precision, allocatable :: swe (:) + double precision, allocatable :: snow_depth (:) + double precision, allocatable :: active_snow_layers (:) + double precision, allocatable :: swe_previous (:) + double precision, allocatable :: snow_soil_interface(:,:) + double precision, allocatable :: temperature_snow (:,:) + double precision, allocatable :: snow_ice_layer (:,:) + double precision, allocatable :: snow_liq_layer (:,:) + double precision, allocatable :: temperature_soil (:,:) +! needed for IMSaggregate_mod + double precision, allocatable :: vegetation_type(:) +! needed by JEDI to mask out land-ice + double precision, allocatable :: soil_moisture_total(:,:) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid(:,:) + double precision, allocatable :: temperature_ground (:) + end type vector_type + + type tile_type + double precision, allocatable :: swe (:,:,:) + double precision, allocatable :: snow_depth (:,:,:) + double precision, allocatable :: active_snow_layers (:,:,:) + double precision, allocatable :: swe_previous (:,:,:) + double precision, allocatable :: snow_soil_interface(:,:,:,:) + double precision, allocatable :: temperature_snow (:,:,:,:) + double precision, allocatable :: snow_ice_layer (:,:,:,:) + double precision, allocatable :: snow_liq_layer (:,:,:,:) + double precision, allocatable :: temperature_soil (:,:,:,:) + real, allocatable :: land_frac (:,:,:) + double precision, allocatable :: soil_moisture_total(:,:,:,:) + double precision, allocatable :: vegetation_type(:,:,:) +! needed by add increments + double precision, allocatable :: slmsk (:, :, :) +! needed for JEDI QC of SMAP data + double precision, allocatable :: soil_moisture_liquid (:,:,:,:) + double precision, allocatable :: temperature_ground (:,:,:) + end type tile_type + +contains + + subroutine vector2tile_restart(namelist) + type(namelist_type) :: namelist + type(vector_type) :: vector + type(tile_type) :: tile + character*256 :: vector_filename + character*300 :: tile_filename + character*19 :: date + integer :: vector_length = 0 + integer :: yyyy,mm,dd,hh,nn,ss + integer :: itile, ix, iy, iloc + integer :: ncid, dimid, varid, status + logical :: file_exists + read(namelist%restart_date( 1: 4),'(i4.4)') yyyy + read(namelist%restart_date( 6: 7),'(i2.2)') mm + read(namelist%restart_date( 9:10),'(i2.2)') dd + read(namelist%restart_date(12:13),'(i2.2)') hh + read(namelist%restart_date(15:16),'(i2.2)') nn + read(namelist%restart_date(18:19),'(i2.2)') ss + + write(date,'(i4,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2,a1,i2.2)') & + yyyy, "-", mm, "-", dd, "_", hh, "-", nn, "-", ss + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate tile variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(tile%swe (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%snow_depth (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%active_snow_layers (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%swe_previous (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%snow_soil_interface(namelist%tile_size,namelist%tile_size,7,6)) + allocate(tile%temperature_snow (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%snow_ice_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%snow_liq_layer (namelist%tile_size,namelist%tile_size,3,6)) + allocate(tile%temperature_soil (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%soil_moisture_total (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%land_frac (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%slmsk (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%vegetation_type (namelist%tile_size,namelist%tile_size,6)) + allocate(tile%soil_moisture_liquid (namelist%tile_size,namelist%tile_size,4,6)) + allocate(tile%temperature_ground (namelist%tile_size,namelist%tile_size,6)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read FV3 tile information +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a5,i1,a3)') ".tile", itile, ".nc" + + tile_filename = trim(namelist%tile_path)//trim(namelist%tile_fstub)//trim(adjustl(tile_filename)) + inquire(file=trim(tile_filename), exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist1" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(trim(tile_filename), NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "land_frac", varid) + status = nf90_get_var(ncid, varid , tile%land_frac(:,:,itile)) + + status = nf90_close(ncid) + + vector_length = vector_length + count(tile%land_frac(:,:,itile) > 0) + + end do + + print*, "The FV3 tiles report ",vector_length, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Allocate vector variables +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + allocate(vector%swe (vector_length)) + allocate(vector%snow_depth (vector_length)) + allocate(vector%active_snow_layers (vector_length)) + allocate(vector%swe_previous (vector_length)) + allocate(vector%snow_soil_interface(vector_length,7)) + allocate(vector%temperature_snow (vector_length,3)) + allocate(vector%snow_ice_layer (vector_length,3)) + allocate(vector%snow_liq_layer (vector_length,3)) + allocate(vector%temperature_soil (vector_length,4)) + allocate(vector%soil_moisture_total (vector_length,4)) + allocate(vector%vegetation_type (vector_length)) + allocate(vector%soil_moisture_liquid (vector_length,4)) + allocate(vector%temperature_ground (vector_length)) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Direction of transfer branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + if(namelist%direction == "vector2tile") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read vector restart file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorRestart(namelist, date, vector, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer vector to tiles +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! explicitly initialize to 0. + tile%slmsk=0. + + iloc = 0 + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + if(tile%land_frac(ix,iy,itile) > 0.0) then + iloc = iloc + 1 + tile%swe(ix,iy,itile) = vector%swe(iloc) + tile%vegetation_type(ix,iy,itile) = vector%vegetation_type(iloc) + tile%snow_depth(ix,iy,itile) = vector%snow_depth(iloc) + tile%active_snow_layers(ix,iy,itile) = vector%active_snow_layers(iloc) + tile%swe_previous(ix,iy,itile) = vector%swe_previous(iloc) + tile%snow_soil_interface(ix,iy,:,itile) = vector%snow_soil_interface(iloc,:) + tile%temperature_snow(ix,iy,:,itile) = vector%temperature_snow(iloc,:) + tile%snow_ice_layer(ix,iy,:,itile) = vector%snow_ice_layer(iloc,:) + tile%snow_liq_layer(ix,iy,:,itile) = vector%snow_liq_layer(iloc,:) + tile%temperature_soil(ix,iy,:,itile) = vector%temperature_soil(iloc,:) + tile%soil_moisture_total(ix,iy,:,itile) = vector%soil_moisture_total(iloc,:) + tile%slmsk(ix,iy,itile) = 1. + tile%soil_moisture_liquid(ix,iy,:,itile)= vector%soil_moisture_liquid(iloc,:) + tile%temperature_ground(ix,iy,itile) = vector%temperature_ground(iloc) + end if + + end do + end do + end do + + print*, "Transferred ",iloc, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write FV3 tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call WriteTileRestart(namelist, date, tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! tile2vector branch +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + elseif(namelist%direction == "tile2vector") then + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read tile restart files +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadTileRestart(namelist, date, tile) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Transfer tile to vector +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + iloc = 0 + do itile = 1, 6 + do iy = 1, namelist%tile_size + do ix = 1, namelist%tile_size + + if(tile%land_frac(ix,iy,itile) > 0.0) then + iloc = iloc + 1 + vector%swe(iloc) = tile%swe(ix,iy,itile) + vector%snow_depth(iloc) = tile%snow_depth(ix,iy,itile) + vector%active_snow_layers(iloc) = tile%active_snow_layers(ix,iy,itile) + vector%swe_previous(iloc) = tile%swe_previous(ix,iy,itile) + vector%snow_soil_interface(iloc,:) = tile%snow_soil_interface(ix,iy,:,itile) + vector%temperature_snow(iloc,:) = tile%temperature_snow(ix,iy,:,itile) + vector%snow_ice_layer(iloc,:) = tile%snow_ice_layer(ix,iy,:,itile) + vector%snow_liq_layer(iloc,:) = tile%snow_liq_layer(ix,iy,:,itile) + vector%temperature_soil(iloc,:) = tile%temperature_soil(ix,iy,:,itile) + vector%soil_moisture_total(iloc,:) = tile%soil_moisture_total(ix,iy,:,itile) + vector%soil_moisture_liquid(iloc,:)= tile%soil_moisture_liquid(ix,iy,:,itile) + vector%temperature_ground(iloc) = tile%temperature_ground(ix,iy,itile) + end if + + end do + end do + end do + + print*, "Transferred ",iloc, "land grids" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write FV3 tile file +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call WriteVectorRestart(namelist, date, vector, vector_length) + + end if ! "vector2tile" or "tile2vector" branch + + end subroutine vector2tile_restart + + subroutine ReadVectorRestart(namelist, date, vector, vector_length) + + use netcdf + + type(namelist_type) :: namelist + type(vector_type) :: vector + character*19 :: date + integer :: vector_length + character*256 :: vector_filename, filename + integer :: ncid, dimid, varid, status + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create vector file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + write(vector_filename,'(a17,a19,a3)') "ufs_land_restart.", date, ".nc" + + filename = trim(namelist%vector_restart_path)//trim(vector_filename) + + inquire(file=filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(filename), " does not exist2" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Reading vector file: ", trim(filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Check the vector length, fail if not consistent with tile-calculated length +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorLength(filename, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the vector fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_varid(ncid, "snow_water_equiv", varid) + if (status /= nf90_noerr) then + print *, 'snow_water_equiv variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%swe , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_depth", varid) + if (status /= nf90_noerr) then + print *, 'snow_depth variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_depth , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "active_snow_levels", varid) + if (status /= nf90_noerr) then + print *, 'active_snow_levels variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%active_snow_layers , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_water_equiv_old", varid) + if (status /= nf90_noerr) then + print *, 'snow_water_equiv_old variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%swe_previous, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "temperature_snow", varid) + if (status /= nf90_noerr) then + print *, 'temperature_snow variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_snow , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "interface_depth", varid) + if (status /= nf90_noerr) then + print *, 'interface_depth variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_soil_interface , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 7, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_ice", varid) + if (status /= nf90_noerr) then + print *, 'snow_level_ice variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_ice_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_liquid", varid) + if (status /= nf90_noerr) then + print *, 'snow_level_liquid variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%snow_liq_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "temperature_soil", varid) + if (status /= nf90_noerr) then + print *, 'temperature_soil variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_soil , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + status = nf90_inq_varid(ncid, "soil_moisture_vol", varid) + if (status /= nf90_noerr) then + print *, 'soil_moisture_vol variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%soil_moisture_total , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + status = nf90_inq_varid(ncid, "soil_liquid_vol", varid) + if (status /= nf90_noerr) then + print *, 'soil_liquid_vol variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%soil_moisture_liquid , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4, 1/)) + + + status = nf90_inq_varid(ncid, "temperature_ground", varid) + if (status /= nf90_noerr) then + print *, 'temperature_ground variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%temperature_ground , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + + ! read vegetation from static file + + filename = trim(namelist%static_filename) + + inquire(file=filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(filename), " does not exist3" + print*, "Check paths and file name" + stop 10 + end if + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_varid(ncid, "vegetation_category", varid) + if (status /= nf90_noerr) then + print *, 'vegetation_category missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , vector%vegetation_type, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + end subroutine ReadVectorRestart + + subroutine ReadTileRestart(namelist, date, tile) + + use netcdf + + type(namelist_type) :: namelist + type(tile_type) :: tile + character*19 :: date + character*256 :: tile_filename + integer :: ncid, dimid, varid, status + integer :: itile + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create tile file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do itile = 1, 6 + write(tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + tile_filename = trim(namelist%tile_restart_path)//trim(tile_filename) + + inquire(file=tile_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(tile_filename), " does not exist4" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Reading tile file: ", trim(tile_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Read the tile fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(tile_filename, NF90_NOWRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Start reading restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + if (status /= nf90_noerr) then + print *, 'sheleg variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + if (status /= nf90_noerr) then + print *, 'snwdph variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + if (status /= nf90_noerr) then + print *, 'snowxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + if (status /= nf90_noerr) then + print *, 'sneqvoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + if (status /= nf90_noerr) then + print *, 'zsnoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + if (status /= nf90_noerr) then + print *, 'tsnoxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + if (status /= nf90_noerr) then + print *, 'snicexy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + if (status /= nf90_noerr) then + print *, 'snliqxy variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + if (status /= nf90_noerr) then + print *, 'stc variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_soil(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + if (status /= nf90_noerr) then + print *, 'smc variable missing from vector file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%soil_moisture_total(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "slc", varid) + if (status /= nf90_noerr) then + print *, 'slc variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + if (status /= nf90_noerr) then + print *, 'tgxy variable missing from tile file' + call handle_err(status) + endif + status = nf90_get_var(ncid, varid , tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine ReadTileRestart + + subroutine WriteVectorRestart(namelist, date, vector, vector_length) + + use netcdf + + type(namelist_type) :: namelist + type(vector_type) :: vector + character*19 :: date + integer :: vector_length + character*256 :: vector_filename + integer :: ncid, dimid, varid, status + logical :: file_exists + +!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Create vector file name +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + write(vector_filename,'(a17,a19,a3)') "ufs_land_restart.", date, ".nc" + + vector_filename = trim(namelist%output_path)//trim(vector_filename) + + inquire(file=vector_filename, exist=file_exists) + + if(.not.file_exists) then + print*, trim(vector_filename), " does not exist5" + print*, "Check paths and file name" + stop 10 + end if + + print*, "Writing vector file: ", trim(vector_filename) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Check the vector length, fail if not consistent with tile-calculated length +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ReadVectorLength(vector_filename, vector_length) + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Write the vector fields +!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + status = nf90_open(vector_filename, NF90_WRITE, ncid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_inq_varid(ncid, "snow_water_equiv", varid) + status = nf90_put_var(ncid, varid , vector%swe , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_depth", varid) + status = nf90_put_var(ncid, varid , vector%snow_depth , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "active_snow_levels", varid) + status = nf90_put_var(ncid, varid , vector%active_snow_layers , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "snow_water_equiv_old", varid) + status = nf90_put_var(ncid, varid , vector%swe_previous, & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_inq_varid(ncid, "temperature_snow", varid) + status = nf90_put_var(ncid, varid , vector%temperature_snow , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "interface_depth", varid) + status = nf90_put_var(ncid, varid , vector%snow_soil_interface , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 7, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_ice", varid) + status = nf90_put_var(ncid, varid , vector%snow_ice_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "snow_level_liquid", varid) + status = nf90_put_var(ncid, varid , vector%snow_liq_layer , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 3, 1/)) + + status = nf90_inq_varid(ncid, "temperature_soil", varid) + status = nf90_put_var(ncid, varid , vector%temperature_soil , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "soil_moisture_vol", varid) + status = nf90_put_var(ncid, varid , vector%soil_moisture_total , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "soil_liquid_vol", varid) + status = nf90_put_var(ncid, varid , vector%soil_moisture_liquid , & + start = (/1 , 1, 1/) , & + count = (/vector_length, 4/)) + + status = nf90_inq_varid(ncid, "temperature_ground", varid) + status = nf90_put_var(ncid, varid , vector%temperature_ground , & + start = (/1,1/), count = (/vector_length, 1/)) + + status = nf90_close(ncid) + + end subroutine WriteVectorRestart + + subroutine WriteTileRestart(namelist, date, tile) + + use netcdf + + type(namelist_type) :: namelist + type(tile_type) :: tile + character*19 :: date + character*256 :: tile_filename + integer :: itile + integer :: ncid, varid, status, i + integer :: dim_id_xdim, dim_id_ydim, dim_id_soil, dim_id_snow, dim_id_snso, dim_id_time + + do itile = 1, 6 + + !write(tile_filename,'(a17,a19,a5,i1,a3)') "ufs_land_restart.", date, ".tile", itile, ".nc" + write(tile_filename,'(a4,a2,a2,a1,a2,a18,i1,a3)') & + date(1:4), date(6:7), date(9:10),".",date(12:13), "0000.sfc_data.tile",itile,".nc" + + tile_filename = trim(namelist%output_path)//trim(tile_filename) + + print*, "Writing tile file: ", trim(tile_filename) + + status = nf90_create(tile_filename, NF90_CLOBBER, ncid) + if (status /= nf90_noerr) call handle_err(status) + +! Define dimensions in the file. + + status = nf90_def_dim(ncid, "xaxis_1" , namelist%tile_size , dim_id_xdim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "yaxis_1" , namelist%tile_size , dim_id_ydim) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_2" , 4 , dim_id_soil) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_3" , 3 , dim_id_snow) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "zaxis_4" , 7 , dim_id_snso) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_def_dim(ncid, "Time" , NF90_UNLIMITED , dim_id_time) + if (status /= nf90_noerr) call handle_err(status) + +! define dimension variables (for JEDI) + + status = nf90_def_var(ncid, "Time", NF90_DOUBLE, & + (/dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "xaxis_1", NF90_DOUBLE, & + (/dim_id_xdim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "yaxis_1", NF90_DOUBLE, & + (/dim_id_ydim/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_2", NF90_DOUBLE, & + (/dim_id_soil/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_3", NF90_DOUBLE, & + (/dim_id_snow/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zaxis_4", NF90_DOUBLE, & + (/dim_id_snso/), varid) + if (status /= nf90_noerr) call handle_err(status) + + +! Define variables in the file. + + status = nf90_def_var(ncid, "sheleg", NF90_DOUBLE, & ! note: this is weasd in vector file. + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snwdph", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snowxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "sneqvoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "zsnsoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snso,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tsnoxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snicexy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "snliqxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_snow,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "stc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "smc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slmsk", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "vtype", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "slc", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_soil,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_def_var(ncid, "tgxy", NF90_DOUBLE, & + (/dim_id_xdim,dim_id_ydim,dim_id_time/), varid) + if (status /= nf90_noerr) call handle_err(status) + + status = nf90_enddef(ncid) + + +! fill dimension variables + + status = nf90_inq_varid(ncid, "Time", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/1/) ) + + status = nf90_inq_varid(ncid, "xaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "yaxis_1", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, namelist%tile_size)/) ) + + status = nf90_inq_varid(ncid, "zaxis_2", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 4)/) ) + + status = nf90_inq_varid(ncid, "zaxis_3", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 3)/) ) + + status = nf90_inq_varid(ncid, "zaxis_4", varid) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_put_var(ncid, varid ,(/(i, i=1, 7)/) ) + +! Start writing restart file + + status = nf90_inq_varid(ncid, "sheleg", varid) + status = nf90_put_var(ncid, varid , tile%swe(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snwdph", varid) + status = nf90_put_var(ncid, varid , tile%snow_depth(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "snowxy", varid) + status = nf90_put_var(ncid, varid , tile%active_snow_layers(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "sneqvoxy", varid) + status = nf90_put_var(ncid, varid , tile%swe_previous(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "zsnsoxy", varid) + status = nf90_put_var(ncid, varid , tile%snow_soil_interface(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 7, 1/)) + + status = nf90_inq_varid(ncid, "tsnoxy", varid) + status = nf90_put_var(ncid, varid , tile%temperature_snow(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snicexy", varid) + status = nf90_put_var(ncid, varid , tile%snow_ice_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "snliqxy", varid) + status = nf90_put_var(ncid, varid , tile%snow_liq_layer(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 3, 1/)) + + status = nf90_inq_varid(ncid, "stc", varid) + status = nf90_put_var(ncid, varid , tile%temperature_soil(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "smc", varid) + status = nf90_put_var(ncid, varid , tile%soil_moisture_total(:,:,:,itile) , & + start = (/1,1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + +! include in output, so can be used to id which tile grid cells are being simulated + status = nf90_inq_varid(ncid, "slmsk", varid) + status = nf90_put_var(ncid, varid , tile%slmsk(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_inq_varid(ncid, "vtype", varid) + status = nf90_put_var(ncid, varid , tile%vegetation_type(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + +! include for JEDI QC of SMAP obs + status = nf90_inq_varid(ncid, "slc", varid) + status = nf90_put_var(ncid, varid , tile%soil_moisture_liquid(:,:,:,itile) , & + start = (/1 , 1 , 1, 1/), & + count = (/namelist%tile_size, namelist%tile_size, 4, 1/)) + + status = nf90_inq_varid(ncid, "tgxy", varid) + status = nf90_put_var(ncid, varid , tile%temperature_ground(:,:,itile) , & + start = (/1,1,1/), count = (/namelist%tile_size, namelist%tile_size, 1/)) + + status = nf90_close(ncid) + + end do + + end subroutine WriteTileRestart + + subroutine ReadVectorLength(filename, vector_length) + + use netcdf + + character*256 :: filename + integer :: vector_length + integer :: length_from_file + integer :: ncid, dimid, varid, status + + status = nf90_open(filename, NF90_NOWRITE, ncid) + + status = nf90_inq_dimid(ncid, "location", dimid) + status = nf90_inquire_dimension(ncid, dimid, len = length_from_file) + + status = nf90_close(ncid) + + if(vector_length /= length_from_file) then + print*, "number of land points in tiles not consistent with land model vector length" + stop 10 + else + print*, "number of land points in tiles consistent with land model vector length" + end if + + end subroutine ReadVectorLength + + subroutine handle_err(status) + use netcdf + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + stop 10 + end if + end subroutine handle_err + +end module vector2tile_restart_mod