diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f058c15639..5061dcea0e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -21,6 +21,7 @@ e4d38681df23ccca0ae29581a45f8362574e0630 025d5e7c2e80263717fb029101d65cbbf261c3c4 a9d96219902cf609636886c7073a84407f450d9a d866510188d26d51bcd6d37239283db690af7e82 +0dcd0a3c1abcaffe5529f8d79a6bc34734b195c7 # Ran SystemTests and python/ctsm through black python formatter 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c 8056ae649c1b37f5e10aaaac79005d6e3a8b2380 diff --git a/.github/workflows/assign-to-project.yml b/.github/workflows/assign-to-project.yml new file mode 100644 index 0000000000..225c223bde --- /dev/null +++ b/.github/workflows/assign-to-project.yml @@ -0,0 +1,23 @@ +name: Auto Assign to Project(s) + +on: + issues: + types: [opened, labeled] + pull_request: + types: [opened, labeled] + issue_comment: + types: [created] + +jobs: + assign_high_priority: + runs-on: ubuntu-latest + name: Assign to High Priority project + steps: + - name: Assign issues and pull requests with priority-high label to project 25 + uses: srggrs/assign-one-project-github-action@1.3.1 + if: | + contains(github.event.issue.labels.*.name, 'priority: high') || + contains(github.event.pull_request.labels.*.name, 'priority: high') + with: + project: 'https://github.com/ESCOMP/CTSM/projects/25' + column_name: 'Needs triage' diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index dc1bc3f0e7..a6fae66356 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -2,7 +2,7 @@ local_path = src/fates protocol = git repo_url = https://github.com/NGEET/fates -tag = sci.1.70.0_api.32.0.0 +tag = sci.1.71.0_api.33.0.0 required = True [externals_description] diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 128b794c83..ed9c4687e9 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -1571,6 +1571,7 @@ sub process_namelist_inline_logic { setup_logic_irrigate($opts, $nl_flags, $definition, $defaults, $nl); setup_logic_start_type($opts, $nl_flags, $nl); setup_logic_decomp_performance($opts, $nl_flags, $definition, $defaults, $nl); + setup_logic_roughness_methods($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_snicar_methods($opts, $nl_flags, $definition, $defaults, $nl); setup_logic_snow($opts, $nl_flags, $definition, $defaults, $nl); setup_logic_glacier($opts, $nl_flags, $definition, $defaults, $nl, $envxml_ref); @@ -1636,7 +1637,7 @@ sub process_namelist_inline_logic { ############################### # namelist group: tillage # ############################### - setup_logic_tillage($opts, $nl_flags, $definition, $defaults, $nl); + setup_logic_tillage($opts, $nl_flags, $definition, $defaults, $nl, $physv); ############################### # namelist group: ch4par_in # @@ -2005,6 +2006,25 @@ sub setup_logic_decomp_performance { #------------------------------------------------------------------------------- +sub setup_logic_roughness_methods { + my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_; + + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'z0param_method', + 'phys'=>$nl_flags->{'phys'} ); + + my $var = remove_leading_and_trailing_quotes( $nl->get_value("z0param_method") ); + if ( $var ne "Meier2022" && $var ne "ZengWang2007" ) { + $log->fatal_error("$var is incorrect entry for the namelist variable z0param_method; expected Meier2022 or ZengWang2007"); + } + my $phys = $physv->as_string(); + if ( $phys eq "clm4_5" || $phys eq "clm5_0" ) { + if ( $var eq "Meier2022" ) { + $log->fatal_error("z0param_method = $var and phys = $phys, but this method has been tested only with clm5_1 and later versions; to use with earlier versions, disable this error, and add Meier2022 parameters to the corresponding params file"); + } + } +} +#------------------------------------------------------------------------------- + sub setup_logic_snicar_methods { my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; @@ -2246,6 +2266,7 @@ sub setup_logic_crop_inparm { 'use_crop'=>$nl->get_value('use_crop') ); my $crop_residue_removal_frac = $nl->get_value('crop_residue_removal_frac'); + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'crop_residue_removal_frac' ); if ( $crop_residue_removal_frac < 0.0 or $crop_residue_removal_frac > 1.0 ) { $log->fatal_error("crop_residue_removal_frac must be in range [0, 1]"); } @@ -2258,10 +2279,13 @@ sub setup_logic_crop_inparm { #------------------------------------------------------------------------------- sub setup_logic_tillage { - my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; + my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_; + + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'tillage_mode', + 'use_crop'=>$nl_flags->{'use_crop'}, 'phys'=>$physv->as_string() ); my $tillage_mode = remove_leading_and_trailing_quotes( $nl->get_value( "tillage_mode" ) ); - if ( $tillage_mode ne "off" && $tillage_mode ne "" && not &value_is_true($nl->get_value('use_crop')) ) { + if ( $tillage_mode ne "off" && $tillage_mode ne "" && not &value_is_true($nl_flags->{'use_crop'}) ) { $log->fatal_error( "Tillage only works on crop columns, so use_crop must be true if tillage is enabled." ); } } diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index f3be69eafe..2ec5f7a5be 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -503,6 +503,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). ZengWang2007 +Meier2022 .true. .false. @@ -558,6 +559,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). .true. 0.d+0 +0.5d00 constant @@ -2037,6 +2039,8 @@ lnd/clm2/surfdata_esmf/NEON/surfdata_1x1_NEON_TOOL_hist_78pfts_CMIP6_simyr2000_c off +low + .false. 0.26d00 diff --git a/cime_config/buildlib b/cime_config/buildlib index 4e3a89f1fd..0e253c9d98 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -140,6 +140,7 @@ def _main_func(): os.path.join(lnd_root, "src", "fates", "biogeochem"), os.path.join(lnd_root, "src", "fates", "fire"), os.path.join(lnd_root, "src", "fates", "parteh"), + os.path.join(lnd_root, "src", "fates", "radiation"), os.path.join(lnd_root, "src", "utils"), os.path.join(lnd_root, "src", "cpl"), os.path.join(lnd_root, "src", "cpl", "utils"), diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index a24299855e..55d1363e6b 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -150,6 +150,13 @@ + + + FAIL + #2321 + + + FAIL @@ -221,4 +228,25 @@ + + + FAIL + #2325 + + + + + + FAIL + #2325 + + + + + + FAIL + #2325 + + + diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index ef5cac6f07..4d4d984980 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -2792,6 +2792,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -3384,13 +3404,13 @@ - + - - + + - + @@ -3713,6 +3733,7 @@ + diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm index e3d311efd4..7f5ece27c8 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm @@ -17,7 +17,7 @@ hist_fincl1 = 'FATES_CROWNAREA_PF', 'FATES_CANOPYCROWNAREA_PF', 'FATES_FABI_SUN_CLLLPF', 'FATES_FABI_SHA_CLLLPF', 'FATES_FABD_SUN_CLLL', 'FATES_FABD_SHA_CLLL', 'FATES_FABI_SUN_CLLL', 'FATES_FABI_SHA_CLLL', 'FATES_PARPROF_DIR_CLLLPF', 'FATES_PARPROF_DIF_CLLLPF', -'FATES_PARPROF_DIR_CLLL', 'FATES_PARPROF_DIF_CLLL', 'FATES_FABD_SUN_TOPLF_CL', +'FATES_FABD_SUN_TOPLF_CL', 'FATES_FABD_SHA_TOPLF_CL', 'FATES_FABI_SUN_TOPLF_CL', 'FATES_FABI_SHA_TOPLF_CL', 'FATES_NET_C_UPTAKE_CLLL', 'FATES_CROWNAREA_CLLL', 'FATES_NPLANT_CANOPY_SZAP', 'FATES_NPLANT_USTORY_SZAP', 'FATES_DDBH_CANOPY_SZAP', 'FATES_DDBH_USTORY_SZAP', diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/shell_commands b/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/shell_commands index 5d230dc5e9..94a832af25 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/shell_commands @@ -1,11 +1,11 @@ SRCDIR=`./xmlquery SRCROOT --value` CASEDIR=`./xmlquery CASEROOT --value` -FATESROOT=$SRCDIR/src/fates/ -FATESPARAMFILE=$FATESROOT/parameter_files/fates_params_seeddisp_4x5.nc +FATESDIR=$SRCDIR/src/fates/ +FATESPARAMFILE=$SRCDIR/fates_params_seeddisp_4x5.nc -ncgen -o $FATESPARAMFILE $FATESROOT/parameter_files/fates_params_default.cdl +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl -$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_fraction --val 0.2 --allpfts -$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_max_dist --val 2500000 --allpfts -$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_scale --val 1e-05 --allpfts -$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_shape --val 0.1 --allpfts +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_fraction --val 0.2 --allpfts +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_max_dist --val 2500000 --allpfts +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_scale --val 1e-05 --allpfts +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_shape --val 0.1 --allpfts diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/user_nl_clm index 8e60c6a2e0..e8d24253c1 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdSeedDisp/user_nl_clm @@ -1,3 +1,3 @@ -fates_paramfile = '$SRCROOT/src/fates/parameter_files/fates_params_seeddisp_4x5.nc' +fates_paramfile = '$SRCROOT/fates_params_seeddisp_4x5.nc' fates_seeddisp_cadence = 1 hist_fincl1 = 'FATES_SEEDS_IN_GRIDCELL_PF', 'FATES_SEEDS_OUT_GRIDCELL_PF' diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/README b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/README new file mode 100644 index 0000000000..295f8125f3 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/README @@ -0,0 +1,15 @@ +Testing FATES two-stream radiation scheme is activated by switching the fates_rad_model +parameter from 1 to 2. This is all that is needed, both radiation schemes +1) Norman and 2) two-stream use the same optical parameters. + +fates_rad_model + +Note that to avoid exceeding the filename string length maximum, the parameter +file generated on the fly is placed in the $SRCROOT/src/fates/parameter_files +directory. This may still run into problems is the $SRCROOT string is too long. + +Like the test with seed dispersal activation, the main downside of this method is +that this file will require a custom update for every fates parameter file API update. +Addressing CTSM issue #2126 will alleviate +this issue as it will provide the capability to build the fates parameter file on +the fly which with the appropriate values for this test. diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/include_user_mods new file mode 100644 index 0000000000..14f7591b72 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/include_user_mods @@ -0,0 +1 @@ +../FatesCold diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/shell_commands b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/shell_commands new file mode 100644 index 0000000000..5d94e5f659 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/shell_commands @@ -0,0 +1,8 @@ +SRCDIR=`./xmlquery SRCROOT --value` +CASEDIR=`./xmlquery CASEROOT --value` +FATESDIR=$SRCDIR/src/fates +FATESPARAMFILE=$CASEDIR/fates_params_twostream.nc + +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl + +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_rad_model --val 2 --allpfts diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/user_nl_clm new file mode 100644 index 0000000000..cae5fc2112 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStream/user_nl_clm @@ -0,0 +1 @@ +fates_paramfile = '$CASEROOT/fates_params_twostream.nc' diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/README b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/README new file mode 100644 index 0000000000..d2c2269fae --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/README @@ -0,0 +1,3 @@ +This tests two-stream radiation crossed with fixed biogeography and nocomp, for +a description of how two-stream is turned on for tests see +FatesColdTwoStream/README diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/include_user_mods new file mode 100644 index 0000000000..17d5840e8c --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/include_user_mods @@ -0,0 +1 @@ +../FatesColdNoComp diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/shell_commands b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/shell_commands new file mode 100644 index 0000000000..5d94e5f659 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/shell_commands @@ -0,0 +1,8 @@ +SRCDIR=`./xmlquery SRCROOT --value` +CASEDIR=`./xmlquery CASEROOT --value` +FATESDIR=$SRCDIR/src/fates +FATESPARAMFILE=$CASEDIR/fates_params_twostream.nc + +ncgen -o $FATESPARAMFILE $FATESDIR/parameter_files/fates_params_default.cdl + +$FATESDIR/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_rad_model --val 2 --allpfts diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/user_nl_clm new file mode 100644 index 0000000000..362dfa4a5e --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdTwoStreamNoCompFixedBioGeo/user_nl_clm @@ -0,0 +1,2 @@ +fates_paramfile = '$CASEROOT/fates_params_twostream.nc' +use_fates_fixed_biogeog=.true. \ No newline at end of file diff --git a/doc/ChangeLog b/doc/ChangeLog index 05888b353a..dbb9b05c84 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,257 @@ =============================================================== +Tag name: ctsm5.1.dev166 +Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310), tking (Teagan King), samrabin (Sam Rabin) +Date: Wed 24 Jan 2024 05:39:41 PM MST +One-line Summary: BFB merge tag + +Purpose and description of changes +---------------------------------- + + #2315 @TeaganKing Refactoring run_neon for PLUMBER2 part1 + #2213 @samsrabin Automatically assign high priority items to project 25 + #2330 @samsrabin Add Izumi version of the aux_clm unit testing + #2326 @samsrabin run_sys_tests: Check Python environment for FatesColdTwoStream tests + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +CTSM issues fixed (include CTSM Issue #): + Fixes #2315 + Fixes #2213 + Fixes #2330 + Fixes #2326 + +Known bugs introduced in this tag (include issue #): + - New feature coming in with #2213 where user will receive email from + github when pushing to their remote: + "Run failed: .github/workflows/assign-to-project.yml" + - New feature that also affects older tags: The izumi FatesColdTwoStream + test submitted from ./run_sys_tests will fail at CREATE_NEWCASE unless users + introduce "module load lang/python/3.7.0" in their .bash_profile. + Longterm solution discussed in #2335. The test also works when submitted + manually with ./create_test. + +Notes of particular relevance for developers: +--------------------------------------------- +Changes to tests or testing: + #2315 New unit tests for arg_parse and NeonSite + #2330 New test in aux_clm that does unit testing on izumi because unit + testing does not work on derecho, yet + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + derecho - OK, pylint gives long list of warnings (expected) + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- OK + izumi ------- OK + + +Answer changes +-------------- +Changes answers relative to baseline: No + +Other details +------------- +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/ctsm/pull/2334 + +=============================================================== +=============================================================== +Tag name: ctsm5.1.dev165 +Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310), oleson (Keith Oleson), samrabin (Sam Rabin) +Date: Fri 19 Jan 2024 06:40:36 PM MST +One-line Summary: Turn Meier2022, tillage, and residue removal on for ctsm5.1, fix #2212 + +Purpose and description of changes +---------------------------------- + +Answer-changing merge-tag: +- Turn Meier2022 on for ctsm5.1. Had turned off temporarily while fixing a bug. +- Bring in Urban answer fix #2212. +- Turn tillage and residue removal on for ctsm5.1. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[x] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- +CTSM issues fixed (include CTSM Issue #): +Fixes #2212 + +Notes of particular relevance for users +--------------------------------------- +Changes made to namelist defaults (e.g., changed parameter values): +- Making Meier2022 the default for ctsm5.1 again. +- Making tillage low by default for ctsm5.1. +- Making residue removal 0.5 by default for ctsm5.1. + +Testing summary: +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- OK + izumi ------- OK + +Answer changes +-------------- + +Changes answers relative to baseline: YES + + [ If a tag changes answers relative to baseline comparison the + following should be filled in (otherwise remove this section). + And always remove these three lines and parts that don't apply. ] + + Summarize any changes to answers, i.e., + - what code configurations: ALL + - what platforms/compilers: ALL + - nature of change:i + clm45 and clm50: larger than roundoff + clm51: possibly climate changing + Effect of Meier2022 was documented here: https://github.com/NCAR/LMWG_dev/issues/38 + Effect of tillage and residue removal may require an Answer Changing Tag simulation + +Other details +------------- +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/ctsm/pull/2323 + +=============================================================== +=============================================================== +Tag name: ctsm5.1.dev164 +Originator(s): rgknox (Ryan Knox) +Date: Wed 17 Jan 2024 12:38:18 PM MST +One-line Summary: Compatibility and tests for FATES 2-Stream + +Purpose and description of changes +---------------------------------- + +This set of changes enables compatibility and testing for FATES two-stream radiation scattering. Two stream radiation is selected by setting the FATES parameter file variable fates_rad_mod = 2. This is an alternative to Norman radiation. The FATES default radiation model will continue to be Norman for the time being, but is expected to transition to two-stream in the near future. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +CTSM issues fixed (include CTSM Issue #): 2305 + +Known bugs introduced in this tag (include issue #): none, but testing was modified to catch a pre-existing bug via test: SMS_D_Ld3.f09_g17.I2000Clm51FatesSpCruRsGs.derecho_gnu.clm-FatesColdSatPhen_prescribed. This has been documented in CTSM issue #2321 + +Notes of particular relevance for users +--------------------------------------- + +Caveats for users (e.g., need to interpolate initial conditions): none + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): none + +Changes made to namelist defaults (e.g., changed parameter values): none + +Changes to the datasets (e.g., parameter, surface or initial files): none + +Substantial timing or memory changes: +[e.g., check PFS test in the test suite and look at timings, if you +expect possible significant timing changes] + +If a fates user does opt to use two-stream radiation, they should expect changes in simulation time compared with norman radiation. This difference varies and is somewhere between equal or 20% slower at a maximum. Most tests seemed to be about 10-15% slower for regions with high vegetation demographic diversity. + +Notes of particular relevance for developers: +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + +Changes to tests or testing: + +New tests were added to the fates and aux_clm regression suites, with suffix clm-FatesColdTwoStream. One of them uses fixed giogeography and no cross-pft competition. + +Testing summary: +---------------- + + regular tests, baseline: ctsm5.1.dev163 + + derecho ----- OK + izumi ------- OK + + fates tests: baseline: fates-sci.1.70.0_api.32.0.0-ctsm5.1.dev163 + derecho ----- OK + izumi ------- (being run after tag was made) + + any other testing (give details below): + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +All answers are B4B with baselines mentioned above, except for one fates variable: FATES_RAD_ERROR. This variable was changed to report the maximum of VIS and NIR, instead of just VIS. A follow up set of changes will change the dimension of this variable. This change was expected. + + +Other details +------------- + +This set of changes is synchronized with the new FATES tag: sci.1.71.0_api.33.0.0 +PR: https://github.com/NGEET/fates/pull/1141 + + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev163 Originator(s): samrabin (Sam Rabin, UCAR/TSS, samrabin@ucar.edu) Date: Wed Jan 10 13:03:34 MST 2024 diff --git a/doc/ChangeSum b/doc/ChangeSum index 50bc77377f..bfc8b86174 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,9 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev163 sam 01/10/2024 Add tillage and residue removal + ctsm5.1.dev166 multiple 01/24/2024 BFB merge tag + ctsm5.1.dev165 slevis 01/19/2024 Turn Meier2022, tillage, residue removal on for ctsm5.1, fix #2212 + ctsm5.1.dev164 rgknox 01/17/2024 Compatibility and tests for FATES 2-Stream + ctsm5.1.dev163 samrabin 01/10/2024 Add tillage and residue removal ctsm5.1.dev162 samrabin 01/05/2024 Improvements to processing of crop calendar files ctsm5.1.dev161 samrabin 01/04/2024 Refactor 20-year running means of crop GDD accumulation ctsm5.1.dev160 glemieux 12/30/2023 FATES landuse version 1 diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index e4a0bcf009..de93081504 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -249,7 +249,7 @@ def run_sys_tests( else: raise RuntimeError("None of suite_name, testfile or testlist were provided") if not running_ctsm_py_tests: - _try_systemtests(testname_list) + _check_py_env(testname_list) _run_create_test( cime_path=cime_path, test_args=test_args, @@ -708,7 +708,23 @@ def _run_test_suite( ) -def _try_systemtests(testname_list): +def _get_testmod_list(test_attributes, unique=False): + # Isolate testmods, producing a list like + # ["clm-test1mod1", "clm-test2mod1", "clm-test2mod2", ...] + # Handles test attributes passed in from run_sys_tests calls using -t, -f, or -s + + testmods = [] + for test_attribute in test_attributes: + for dot_split in test_attribute.split("."): + slash_replaced = dot_split.replace("/", "-") + for ddash_split in slash_replaced.split("--"): + if "clm-" in ddash_split and (ddash_split not in testmods or not unique): + testmods.append(ddash_split) + + return testmods + + +def _check_py_env(test_attributes): err_msg = " can't be loaded. Do you need to activate the ctsm_pylib conda environment?" # Suppress pylint import-outside-toplevel warning because (a) we only want to import # this when certain tests are requested, and (b) the import needs to be in a try-except @@ -716,12 +732,31 @@ def _try_systemtests(testname_list): # pylint: disable=import-outside-toplevel disable # Suppress pylint unused-import warning because the import itself IS the use. # pylint: disable=unused-import disable - if any("FSURDATMODIFYCTSM" in t for t in testname_list): + # Suppress pylint import-error warning because the whole point here is to check + # whether import is possible. + # pylint: disable=import-error disable + + # Check requirements for FSURDATMODIFYCTSM, if needed + if any("FSURDATMODIFYCTSM" in t for t in test_attributes): try: import ctsm.modify_input_files.modify_fsurdat except ModuleNotFoundError as err: raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err + # Check that list for any testmods that use modify_fates_paramfile.py + testmods_to_check = ["clm-FatesColdTwoStream", "clm-FatesColdTwoStreamNoCompFixedBioGeo"] + testmods = _get_testmod_list(test_attributes) + if any(t in testmods_to_check for t in testmods): + # This bit is needed because it's outside the top-level python/ directory. + fates_dir = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "src", "fates" + ) + sys.path.insert(1, fates_dir) + try: + import tools.modify_fates_paramfile + except ModuleNotFoundError as err: + raise ModuleNotFoundError("modify_fates_paramfile" + err_msg) from err + def _get_compilers_for_suite(suite_name, machine_name, running_ctsm_py_tests): test_data = get_tests_from_xml(xml_machine=machine_name, xml_category=suite_name) @@ -730,7 +765,8 @@ def _get_compilers_for_suite(suite_name, machine_name, running_ctsm_py_tests): "No tests found for suite {} on machine {}".format(suite_name, machine_name) ) if not running_ctsm_py_tests: - _try_systemtests([t["testname"] for t in test_data]) + _check_py_env([t["testname"] for t in test_data]) + _check_py_env([t["testmods"] for t in test_data if "testmods" in t.keys()]) compilers = sorted({one_test["compiler"] for one_test in test_data}) logger.info("Running with compilers: %s", compilers) return compilers diff --git a/python/ctsm/site_and_regional/neon_arg_parse.py b/python/ctsm/site_and_regional/neon_arg_parse.py new file mode 100644 index 0000000000..99f184dd62 --- /dev/null +++ b/python/ctsm/site_and_regional/neon_arg_parse.py @@ -0,0 +1,240 @@ +""" +Argument parser to use throughout run_neon.py +""" + +import argparse +import logging +import os +import sys + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order +from ctsm import add_cime_to_path +from ctsm.utils import parse_isoduration +from CIME.utils import parse_args_and_handle_standard_logging_options +from CIME.utils import setup_standard_logging_options + + +def get_parser(args, description, valid_neon_sites): + """ + Get parser object for this script. + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + setup_standard_logging_options(parser) + + parser.print_usage = parser.print_help + + parser.add_argument( + "--neon-sites", + help="4-letter neon site code.", + action="store", + required=False, + choices=valid_neon_sites + ["all"], + dest="neon_sites", + default=["OSBS"], + nargs="+", + ) + + parser.add_argument( + "--base-case", + help=""" + Root Directory of base case build + [default: %(default)s] + """, + action="store", + dest="base_case_root", + type=str, + required=False, + default=None, + ) + + parser.add_argument( + "--output-root", + help=""" + Root output directory of cases + [default: %(default)s] + """, + action="store", + dest="output_root", + type=str, + required=False, + default="CIME_OUTPUT_ROOT as defined in cime", + ) + + parser.add_argument( + "--overwrite", + help=""" + overwrite existing case directories + [default: %(default)s] + """, + action="store_true", + dest="overwrite", + required=False, + default=False, + ) + + parser.add_argument( + "--setup-only", + help=""" + Only setup the requested cases, do not build or run + [default: %(default)s] + """, + action="store_true", + dest="setup_only", + required=False, + default=False, + ) + + parser.add_argument( + "--rerun", + help=""" + If the case exists but does not appear to be complete, restart it. + [default: %(default)s] + """, + action="store_true", + dest="rerun", + required=False, + default=False, + ) + + parser.add_argument( + "--no-batch", + help=""" + Run locally, do not use batch queueing system (if defined for Machine) + [default: %(default)s] + """, + action="store_true", + dest="no_batch", + required=False, + default=False, + ) + + parser.add_argument( + "--run-type", + help=""" + Type of run to do + [default: %(default)s] + """, + choices=["ad", "postad", "transient"], # , "sasu"], + default="transient", + ) + + parser.add_argument( + "--prism", + help=""" + Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data + (only available over Continental US) + """, + action="store_true", + dest="prism", + required=False, + default=False, + ) + + parser.add_argument( + "--experiment", + help=""" + Appends the case name with string for model experiment + """, + action="store", + dest="experiment", + type=str, + required=False, + default=None, + ) + + parser.add_argument( + "--run-length", + help=""" + How long to run (modified ISO 8601 duration) + [default: %(default)s] + """, + required=False, + type=str, + default="0Y", + ) + + parser.add_argument( + "--run-from-postad", + help=""" + For transient runs only - should we start from the postad spinup or finidat? + By default start from finidat, if this flag is used the postad run must be available. + """, + action="store_true", + required=False, + default=False, + ) + parser.add_argument( + "--neon-version", + help=""" + Neon data version to use for this simulation. + [default: use the latest data available] + """, + action="store", + dest="user_version", + required=False, + type=str, + choices=["v1", "v2", "v3"], + ) + + args = parse_args_and_handle_standard_logging_options(args, parser) + + if "all" in args.neon_sites: + neon_sites = valid_neon_sites + else: + neon_sites = args.neon_sites + for site in neon_sites: + if site not in valid_neon_sites: + raise ValueError("Invalid site name {}".format(site)) + + if "CIME_OUTPUT_ROOT" in args.output_root: + args.output_root = None + + if args.run_length == "0Y": + if args.run_type == "ad": + run_length = "100Y" + elif args.run_type == "postad": + run_length = "100Y" + else: + # The transient run length is set by cdeps atm buildnml to + # the last date of the available tower data + # this value is not used + run_length = "4Y" + else: + run_length = args.run_length + + run_length = parse_isoduration(run_length) + + base_case_root = None + if args.base_case_root: + base_case_root = os.path.abspath(args.base_case_root) + if not os.path.exists(base_case_root): + raise ValueError("Base case root does not exist: {}".format(base_case_root)) + + # Reduce output level for this script unless --debug or + # --verbose is provided on the command line + if not args.debug and not args.verbose: + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARN) + + return ( + neon_sites, + args.output_root, + args.run_type, + args.experiment, + args.prism, + args.overwrite, + run_length, + base_case_root, + args.run_from_postad, + args.setup_only, + args.no_batch, + args.rerun, + args.user_version, + ) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py new file mode 100755 index 0000000000..31ae78f5ad --- /dev/null +++ b/python/ctsm/site_and_regional/neon_site.py @@ -0,0 +1,394 @@ +""" +This module contains the NeonSite class and class functions which are used in run_neon.py +""" + +# Import libraries +import glob +import logging +import os +import re +import shutil +import sys +import time + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root + +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, symlink_force + +logger = logging.getLogger(__name__) + + +# pylint: disable=too-many-instance-attributes +class NeonSite: + """ + A class for encapsulating neon sites. + """ + + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): + self.name = name + self.start_year = int(start_year) + self.end_year = int(end_year) + self.start_month = int(start_month) + self.end_month = int(end_month) + self.cesmroot = path_to_ctsm_root() + self.finidat = finidat + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init + self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build(case_path, case=case) + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + + # pylint: disable=no-self-use + def get_batch_query(self, case): + """ + Function for querying the batch queue query command for a case, depending on the + user's batch system. + + Args: + case: + case object + """ + + if case.get_value("BATCH_SYSTEM") == "none": + return "none" + return case.get_value("batch_query") + + # pylint: disable=too-many-statements + def run_case( + self, + base_case_root, + run_type, + prism, + run_length, + user_version, + overwrite=False, + setup_only=False, + no_batch=False, + rerun=False, + experiment=False, + ): + """ + Run case. + + Args: + self + base_case_root: str, opt + file path of base case + run_type: str, opt + transient, post_ad, or ad case, default transient + prism: bool, opt + if True, use PRISM precipitation, default False + run_length: str, opt + length of run, default '4Y' + user_version: str, opt + default 'latest' + overwrite: bool, opt + default False + setup_only: bool, opt + default False; if True, set up but do not run case + no_batch: bool, opt + default False + rerun: bool, opt + default False + experiment: str, opt + name of experiment, default False + """ + user_mods_dirs = [ + os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) + ] + expect( + os.path.isdir(base_case_root), + "Error base case does not exist in {}".format(base_case_root), + ) + # -- if user gives a version: + if user_version: + version = user_version + else: + version = "latest" + + print("using this version:", version) + + if experiment is not None: + self.name = self.name + "." + experiment + case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) + + rundir = None + if os.path.isdir(case_root): + if overwrite: + print("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + # pylint: disable=undefined-variable + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + # pylint: enable=undefined-variable + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): + print("Case {} appears to be complete, not rerunning.".format(case_root)) + elif not setup_only: + print("Resubmitting case {}".format(case_root)) + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") + return + else: + logger.warning("Case already exists in %s, not overwritting", case_root) + return + + if run_type == "postad": + adcase_root = case_root.replace(".postad", ".ad") + if not os.path.isdir(adcase_root): + logger.warning("postad requested but no ad case found in %s", adcase_root) + return + + if not os.path.isdir(case_root): + # read_only = False should not be required here + with Case(base_case_root, read_only=False) as basecase: + print("---- cloning the base case in {}".format(case_root)) + # + # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although + # it causes some of the user_nl_* files to have duplicated inputs. It also ensures + # that the shell_commands file is copied, as well as taking care of the DATM inputs. + # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 + # + basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) + + with Case(case_root, read_only=False) as case: + if run_type != "transient": + # in order to avoid the complication of leap years, + # we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") + case.set_value("REST_OPTION", "end") + case.set_value("CONTINUE_RUN", False) + case.set_value("NEONVERSION", version) + if prism: + case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") + + if run_type == "ad": + case.set_value("CLM_FORCE_COLDSTART", "on") + case.set_value("CLM_ACCELERATED_SPINUP", "on") + case.set_value("RUN_REFDATE", "0018-01-01") + case.set_value("RUN_STARTDATE", "0018-01-01") + case.set_value("RESUBMIT", 1) + case.set_value("STOP_N", run_length) + + else: + case.set_value("CLM_FORCE_COLDSTART", "off") + case.set_value("CLM_ACCELERATED_SPINUP", "off") + case.set_value("RUN_TYPE", "hybrid") + + if run_type == "postad": + self.set_ref_case(case) + case.set_value("STOP_N", run_length) + + # For transient cases STOP will be set in the user_mod_directory + if run_type == "transient": + if self.finidat: + case.set_value("RUN_TYPE", "startup") + else: + if not self.set_ref_case(case): + return + case.set_value("CALENDAR", "GREGORIAN") + case.set_value("RESUBMIT", 0) + case.set_value("STOP_OPTION", "nmonths") + + if not rundir: + rundir = case.get_value("RUNDIR") + + self.modify_user_nl(case_root, run_type, rundir) + + case.create_namelists() + # explicitly run check_input_data + case.check_all_input_data() + if not setup_only: + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") + + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True + + def modify_user_nl(self, case_root, run_type, rundir): + """ + Modify user namelist. If transient, include finidat in user_nl; + Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. + """ + user_nl_fname = os.path.join(case_root, "user_nl_clm") + user_nl_lines = None + if run_type == "transient": + if self.finidat: + user_nl_lines = [ + "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) + ] + else: + user_nl_lines = [ + "hist_fincl2 = ''", + "hist_mfilt = 20", + "hist_nhtfrq = -8760", + "hist_empty_htapes = .true.", + """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', + 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'""", + ] + + if user_nl_lines: + with open(user_nl_fname, "a") as nl_file: + for line in user_nl_lines: + nl_file.write("{}\n".format(line)) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index a69dc0bdb0..72bf3fdfb4 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -47,670 +47,32 @@ # - [ ] Matrix spin-up if (SASU) Eric merged it in # - [ ] Make sure both AD and SASU are not on at the same time -# - [ ] Make sure CIME and other dependencies is checked out. +# - [ ] Make sure CIME and other dependencies are checked out. # Import libraries -import argparse -import datetime import glob import logging import os -import re -import shutil import sys -import time import pandas as pd -from standard_script_setup import * - # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) -from ctsm import add_cime_to_path - -from CIME import build -from CIME.case import Case -from CIME.utils import safe_copy, expect, symlink_force - +# pylint: disable=wrong-import-position from ctsm.path_utils import path_to_ctsm_root -from ctsm.utils import parse_isoduration from ctsm.download_utils import download_file +from ctsm.site_and_regional.neon_arg_parse import get_parser +from ctsm.site_and_regional.neon_site import NeonSite +# pylint: disable=import-error, wildcard-import, wrong-import-order from standard_script_setup import * logger = logging.getLogger(__name__) -def get_parser(args, description, valid_neon_sites): - """ - Get parser object for this script. - """ - parser = argparse.ArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - CIME.utils.setup_standard_logging_options(parser) - - parser.print_usage = parser.print_help - - parser.add_argument( - "--neon-sites", - help="4-letter neon site code.", - action="store", - required=False, - choices=valid_neon_sites + ["all"], - dest="neon_sites", - default=["OSBS"], - nargs="+", - ) - - parser.add_argument( - "--base-case", - help=""" - Root Directory of base case build - [default: %(default)s] - """, - action="store", - dest="base_case_root", - type=str, - required=False, - default=None, - ) - - parser.add_argument( - "--output-root", - help=""" - Root output directory of cases - [default: %(default)s] - """, - action="store", - dest="output_root", - type=str, - required=False, - default="CIME_OUTPUT_ROOT as defined in cime", - ) - - parser.add_argument( - "--overwrite", - help=""" - overwrite existing case directories - [default: %(default)s] - """, - action="store_true", - dest="overwrite", - required=False, - default=False, - ) - - parser.add_argument( - "--setup-only", - help=""" - Only setup the requested cases, do not build or run - [default: %(default)s] - """, - action="store_true", - dest="setup_only", - required=False, - default=False, - ) - - parser.add_argument( - "--rerun", - help=""" - If the case exists but does not appear to be complete, restart it. - [default: %(default)s] - """, - action="store_true", - dest="rerun", - required=False, - default=False, - ) - - parser.add_argument( - "--no-batch", - help=""" - Run locally, do not use batch queueing system (if defined for Machine) - [default: %(default)s] - """, - action="store_true", - dest="no_batch", - required=False, - default=False, - ) - - parser.add_argument( - "--run-type", - help=""" - Type of run to do - [default: %(default)s] - """, - choices=["ad", "postad", "transient", "sasu"], - default="transient", - ) - - parser.add_argument( - "--prism", - help=""" - Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data - (only available over Continental US) - """, - action="store_true", - dest="prism", - required=False, - default=False, - ) - - parser.add_argument( - "--experiment", - help=""" - Appends the case name with string for model experiment - """, - action="store", - dest="experiment", - type=str, - required=False, - default=None, - ) - - parser.add_argument( - "--run-length", - help=""" - How long to run (modified ISO 8601 duration) - [default: %(default)s] - """, - required=False, - type=str, - default="0Y", - ) - - parser.add_argument( - "--start-date", - help=""" - Start date for running CTSM simulation in ISO format. - [default: %(default)s] - (currently non-functional) - """, - action="store", - dest="start_date", - required=False, - type=datetime.date.fromisoformat, - default=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), - ) - - parser.add_argument( - "--end-date", - help=""" - End date for running CTSM simulation in ISO format. - [default: %(default)s] - """, - action="store", - dest="end_date", - required=False, - type=datetime.date.fromisoformat, - default=datetime.datetime.strptime("2021-01-01", "%Y-%m-%d"), - ) - - parser.add_argument( - "--run-from-postad", - help=""" - For transient runs only - should we start from the postad spinup or finidat? - By default start from finidat, if this flag is used the postad run must be available. - """, - action="store_true", - required=False, - default=False, - ) - parser.add_argument( - "--neon-version", - help=""" - Neon data version to use for this simulation. - [default: use the latest data available] - """, - action="store", - dest="user_version", - required=False, - type=str, - choices=["v1", "v2", "v3"], - ) - - args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - - if "all" in args.neon_sites: - neon_sites = valid_neon_sites - else: - neon_sites = args.neon_sites - for site in neon_sites: - if site not in valid_neon_sites: - raise ValueError("Invalid site name {}".format(site)) - - if "CIME_OUTPUT_ROOT" in args.output_root: - args.output_root = None - - if args.run_length == "0Y": - if args.run_type == "ad": - run_length = "100Y" - elif args.run_type == "postad": - run_length = "100Y" - else: - # The transient run length is set by cdeps atm buildnml to - # the last date of the available tower data - # this value is not used - run_length = "4Y" - else: - run_length = args.run_length - - run_length = parse_isoduration(run_length) - base_case_root = None - if args.base_case_root: - base_case_root = os.path.abspath(args.base_case_root) - - # Reduce output level for this script unless --debug or - # --verbose is provided on the command line - if not args.debug and not args.verbose: - root_logger = logging.getLogger() - root_logger.setLevel(logging.WARN) - - return ( - neon_sites, - args.output_root, - args.run_type, - args.experiment, - args.prism, - args.overwrite, - run_length, - base_case_root, - args.run_from_postad, - args.setup_only, - args.no_batch, - args.rerun, - args.user_version, - ) - - -class NeonSite: - """ - A class for encapsulating neon sites. - - ... - - Attributes - ---------- - - Methods - ------- - """ - - def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - self.name = name - self.start_year = int(start_year) - self.end_year = int(end_year) - self.start_month = int(start_month) - self.end_month = int(end_month) - self.cesmroot = path_to_ctsm_root() - self.finidat = finidat - - def __str__(self): - return str(self.__class__) + "\n" + "\n".join((str(item) + " = " for item in self.__dict__)) - - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False - ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - self.base_case_root = output_root - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") - - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - "Existing base case is a historical type and should " - + "not be -- rerun with the --overwrite option", - ) - else: - expect( - match is not None, - "Existing base case should be a historical type and " - + "is not -- rerun with the --overwrite option", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - t_0 = time.time() - build.case_build(case_path, case=case) - t_1 = time.time() - total = t_1 - t_0 - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case - return case_path - - def diff_month(self): - """ - Determine difference between two dates in months - """ - d_1 = datetime.datetime(self.end_year, self.end_month, 1) - d_2 = datetime.datetime(self.start_year, self.start_month, 1) - return (d_1.year - d_2.year) * 12 + d_1.month - d_2.month - - def run_case( - self, - base_case_root, - run_type, - prism, - run_length, - user_version, - overwrite=False, - setup_only=False, - no_batch=False, - rerun=False, - experiment=False, - ): - """ - Run case. - - Args: - self - base_case_root: str, opt - file path of base case - run_type: str, opt - transient, post_ad, or ad case, default transient - prism: bool, opt - if True, use PRISM precipitation, default False - run_length: str, opt - length of run, default '4Y' - user_version: str, opt - default 'latest' - overwrite: bool, opt - default False - setup_only: bool, opt - default False; if True, set up but do not run case - no_batch: bool, opt - default False - rerun: bool, opt - default False - experiment: str, opt - name of experiment, default False - """ - user_mods_dirs = [ - os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] - expect( - os.path.isdir(base_case_root), - "Error base case does not exist in {}".format(base_case_root), - ) - # -- if user gives a version: - if user_version: - version = user_version - else: - version = "latest" - - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - "Existing base case is a historical type and " - + "should not be -- rerun with the --overwrite option", - ) - else: - expect( - match is not None, - "Existing base case should be a historical type " - + "and is not -- rerun with the --overwrite option", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting.", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - mon = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if mon: - refdate = mon.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", - "hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', " - + "'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', " - + "'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'", - ] - - if user_nl_lines: - with open(user_nl_fname, "a") as f_d: - for line in user_nl_lines: - f_d.write("{}\n".format(line)) - - -def get_batch_query(case): - """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. - - Args: - case: - case object - """ - - if case.get_value("BATCH_SYSTEM") == "none": - return "none" - return case.get_value("batch_query") - - def check_neon_listing(valid_neon_sites): """ A function to download and parse neon listing file. @@ -742,19 +104,19 @@ def parse_neon_listing(listing_file, valid_neon_sites): available_list = [] - d_f = pd.read_csv(listing_file) + listing_df = pd.read_csv(listing_file) # check for finidat files for transient run - finidatlist = d_f[d_f["object"].str.contains("lnd/ctsm")] + finidatlist = listing_df[listing_df["object"].str.contains("lnd/ctsm")] # -- filter lines with atm/cdep - d_f = d_f[d_f["object"].str.contains("atm/cdeps/")] + listing_df = listing_df[listing_df["object"].str.contains("atm/cdeps/")] # -- split the object str to extract site name - d_f = d_f["object"].str.split("/", expand=True) + listing_df = listing_df["object"].str.split("/", expand=True) # -- groupby site name - grouped_df = d_f.groupby(8) + grouped_df = listing_df.groupby(8) for key, _ in grouped_df: # -- check if it is a valid neon site if any(key in x for x in valid_neon_sites): @@ -787,7 +149,7 @@ def parse_neon_listing(listing_file, valid_neon_sites): start_month = tmp_df2[1].iloc[0] end_month = tmp_df2[1].iloc[-1] - logger.debug("Valid neon site found: %s", site_name) + logger.debug("Valid neon site %s found!", site_name) logger.debug("File version %s", latest_version) logger.debug("start_year=%s", start_year) logger.debug("end_year=%s", end_year) diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py new file mode 100755 index 0000000000..7bae337709 --- /dev/null +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Unit tests for neon_arg_parse + +You can run this by: + python -m unittest test_unit_neon_arg_parse.py +""" + +import unittest +import tempfile +import shutil +import os +import sys +import glob + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.neon_arg_parse import get_parser +from ctsm.path_utils import path_to_ctsm_root + +# pylint: disable=invalid-name + + +class Test_neon_arg_parse(unittest.TestCase): + """ + Basic class for testing neon_arg_parse.py. + """ + + def setUp(self): + """ + Make /_tempdir for use by these tests. + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_function(self): + """ + Test that neon_arg_parse is properly reading arguments... + """ + sys.argv = [ + "neon_arg_parse", + "--neon-sites", + "ABBY", + "--experiment", + "test", + "--run-type", + "ad", + ] + description = "" + cesmroot = path_to_ctsm_root() + valid_neon_sites = glob.glob( + os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") + ) + valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) + parsed_arguments = get_parser(sys.argv, description, valid_neon_sites) + + self.assertEqual(parsed_arguments[0][0], "ABBY", "arguments not processed as expected") + self.assertEqual(parsed_arguments[3], "test", "arguments not processed as expected") + self.assertEqual(parsed_arguments[4], False, "arguments not processed as expected") + self.assertEqual(parsed_arguments[2], "ad", "arguments not processed as expected") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py new file mode 100755 index 0000000000..4828718272 --- /dev/null +++ b/python/ctsm/test/test_unit_neon_site.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +Unit tests for NeonSite + +You can run this by: + python -m unittest test_unit_neon_site.py +""" + +import unittest +import tempfile +import shutil +import os +import glob +import sys + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.site_and_regional.neon_site import NeonSite + +# pylint: disable=invalid-name + + +class TestNeonSite(unittest.TestCase): + """ + Basic class for testing NeonSite.py. + """ + + def setUp(self): + """ + Make /_tempdir for use by these tests. + """ + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_modify_user_nl_transient(self): + """ + Test that modify_user_nl is correctly adding lines to namelist for transient cases + """ + # NeonSite parameters: + name = "ABBY" + start_year = 2020 + end_year = 2021 + start_month = 1 + end_month = 12 + # finidat = None + finidat = "dummy_finidat" + + # modify_user_nl parameters: + case_root = self._tempdir + run_type = "transient" + rundir = "" + + # create NeonSite object and update namelist + NeonSite(name, start_year, end_year, start_month, end_month, finidat).modify_user_nl( + case_root, run_type, rundir + ) + + # gather file contents for test + new_nl_file = open(glob.glob(case_root + "/*")[0], "r") + lines_read = new_nl_file.readlines()[0] + new_nl_file.close() + + # assertion + self.assertEqual( + lines_read, + "finidat = '/inputdata/lnd/ctsm/initdata/dummy_finidat'\n", + "transient case has unexpected nl", + ) + + def test_modify_user_nl_ad(self): + """ + Test that modify_user_nl is correctly adding lines to namelist for ad cases + """ + # NeonSite parameters: + name = "ABBY" + start_year = 2020 + end_year = 2021 + start_month = 1 + end_month = 12 + # finidat = None + finidat = "dummy_finidat" + + # modify_user_nl parameters: + case_root = self._tempdir + run_type = "ad" + rundir = "" + + # create NeonSite object and update namelist + NeonSite(name, start_year, end_year, start_month, end_month, finidat).modify_user_nl( + case_root, run_type, rundir + ) + + # gather file contents for test + new_nl_file = open(glob.glob(case_root + "/*")[0], "r") + lines_read = new_nl_file.readlines()[1] + new_nl_file.close() + + # assertion + self.assertEqual(lines_read, "hist_mfilt = 20\n", "ad case has unexpected nl") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_run_sys_tests.py b/python/ctsm/test/test_unit_run_sys_tests.py index ee5197d76f..65ec1df5a5 100755 --- a/python/ctsm/test/test_unit_run_sys_tests.py +++ b/python/ctsm/test/test_unit_run_sys_tests.py @@ -16,7 +16,7 @@ from ctsm import add_cime_to_path # pylint: disable=unused-import from ctsm import unit_testing -from ctsm.run_sys_tests import run_sys_tests +from ctsm.run_sys_tests import run_sys_tests, _get_testmod_list from ctsm.machine_defaults import MACHINE_DEFAULTS from ctsm.machine import create_machine from ctsm.joblauncher.job_launcher_factory import JOB_LAUNCHER_FAKE @@ -269,6 +269,57 @@ def test_withDryRun_nothingDone(self): self.assertEqual(os.listdir(self._scratch), []) self.assertEqual(machine.job_launcher.get_commands(), []) + def test_getTestmodList_suite(self): + """Ensure that _get_testmod_list() works correctly with suite-style input""" + input = [ + "clm/default", + "clm/default", + "clm/crop", + "clm/cropMonthlyOutput", + ] + target = [ + "clm-default", + "clm-default", + "clm-crop", + "clm-cropMonthlyOutput", + ] + output = _get_testmod_list(input, unique=False) + self.assertEqual(output, target) + + def test_getTestmodList_suite_unique(self): + """Ensure that _get_testmod_list() works correctly with unique=True""" + input = [ + "clm/default", + "clm/default", + "clm/crop", + "clm/cropMonthlyOutput", + ] + target = [ + "clm-default", + "clm-crop", + "clm-cropMonthlyOutput", + ] + + output = _get_testmod_list(input, unique=True) + self.assertEqual(output, target) + + def test_getTestmodList_testname(self): + """Ensure that _get_testmod_list() works correctly with full test name(s) specified""" + input = [ + "ERS_D_Ld15.f45_f45_mg37.I2000Clm50FatesRs.izumi_nag.clm-crop", + "ERS_D_Ld15.f45_f45_mg37.I2000Clm50FatesRs.izumi_nag.clm-default", + ] + target = ["clm-crop", "clm-default"] + output = _get_testmod_list(input) + self.assertEqual(output, target) + + def test_getTestmodList_twomods(self): + """Ensure that _get_testmod_list() works correctly with full test name(s) specified and two mods in one test""" + input = ["ERS_D_Ld15.f45_f45_mg37.I2000Clm50FatesRs.izumi_nag.clm-default--clm-crop"] + target = ["clm-default", "clm-crop"] + output = _get_testmod_list(input) + self.assertEqual(output, target) + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/src/biogeophys/UrbBuildTempOleson2015Mod.F90 b/src/biogeophys/UrbBuildTempOleson2015Mod.F90 index bf8b68c7eb..4c985f0ab3 100644 --- a/src/biogeophys/UrbBuildTempOleson2015Mod.F90 +++ b/src/biogeophys/UrbBuildTempOleson2015Mod.F90 @@ -383,9 +383,11 @@ subroutine BuildingTemperature (bounds, num_urbanl, filter_urbanl, num_nolakec, ! Get terms from soil temperature equations to compute conduction flux ! Negative is toward surface - heat added - ! Note that the conduction flux here is in W m-2 wall area but for purposes of solving the set of - ! simultaneous equations this must be converted to W m-2 floor area. This is done below when - ! setting up the equation coefficients. + ! Note that the convection and conduction fluxes for the walls are in W m-2 wall area + ! but for purposes of solving the set of simultaneous equations this must be converted to W m-2 + ! floor or roof area. This is done below when setting up the equation coefficients by multiplying by building_hwr. + ! Note also that the longwave radiation terms for the walls are in terms of W m-2 floor area since the view + ! factors implicitly convert from per unit wall area to per unit floor or roof area. do fc = 1,num_nolakec c = filter_nolakec(fc) @@ -424,10 +426,8 @@ subroutine BuildingTemperature (bounds, num_urbanl, filter_urbanl, num_nolakec, ! This view factor implicitly converts from per unit wall area to per unit floor area vf_wf(l) = 0.5_r8*(1._r8 - vf_rf(l)) - ! This view factor implicitly converts from per unit floor area to per unit wall area - vf_fw(l) = vf_wf(l) / building_hwr(l) + vf_fw(l) = vf_wf(l) - ! This view factor implicitly converts from per unit roof area to per unit wall area vf_rw(l) = vf_fw(l) ! This view factor implicitly converts from per unit wall area to per unit roof area @@ -831,7 +831,7 @@ subroutine BuildingTemperature (bounds, num_urbanl, filter_urbanl, num_nolakec, + em_floori(l)*sb*t_floor_bef(l)**4._r8 & + 4._r8*em_floori(l)*sb*t_floor_bef(l)**3.*(t_floor(l) - t_floor_bef(l)) - qrd_building(l) = qrd_roof(l) + building_hwr(l)*(qrd_sunw(l) + qrd_shdw(l)) + qrd_floor(l) + qrd_building(l) = qrd_roof(l) + qrd_sunw(l) + qrd_shdw(l) + qrd_floor(l) if (abs(qrd_building(l)) > .10_r8 ) then write (iulog,*) 'urban inside building net longwave radiation balance error ',qrd_building(l) diff --git a/src/main/controlMod.F90 b/src/main/controlMod.F90 index b3740086e8..eadc45e226 100644 --- a/src/main/controlMod.F90 +++ b/src/main/controlMod.F90 @@ -790,7 +790,7 @@ subroutine control_spmd() call mpi_bcast (use_fates_bgc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (fates_inventory_ctrl_filename, len(fates_inventory_ctrl_filename), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (fates_paramfile, len(fates_paramfile) , MPI_CHARACTER, 0, mpicom, ier) - call mpi_bcast (fluh_timeseries, len(fates_paramfile) , MPI_CHARACTER, 0, mpicom, ier) + call mpi_bcast (fluh_timeseries, len(fluh_timeseries) , MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (fates_parteh_mode, 1, MPI_INTEGER, 0, mpicom, ier) call mpi_bcast (fates_seeddisp_cadence, 1, MPI_INTEGER, 0, mpicom, ier) diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index 83d7186021..7039884847 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -150,7 +150,7 @@ module CLMFatesInterfaceMod use EDInitMod , only : init_patches use EDInitMod , only : set_site_properties use EDPftVarcon , only : EDpftvarcon_inst - use EDSurfaceRadiationMod , only : ED_SunShadeFracs, ED_Norman_Radiation + use FatesRadiationDriveMod, only : FatesSunShadeFracs, FatesNormalizedCanopyRadiation use EDBtranMod , only : btran_ed, & get_active_suction_layers use EDCanopyStructureMod , only : canopy_summarization, update_hlm_dynamics @@ -1147,7 +1147,7 @@ subroutine dynamics_driv(this, nc, bounds_clump, & call fates_hist%update_history_dyn( nc, & this%fates(nc)%nsites, & this%fates(nc)%sites, & - this%fates(nc)%bc_in) + this%fates(nc)%bc_in ) if (masterproc) then write(iulog, *) 'clm: leaving fates model', bounds_clump%begg, & @@ -2150,7 +2150,7 @@ subroutine wrap_sunfrac(this,nc,atm2lnd_inst,canopystate_inst) ! as well as total patch sun/shade fraction output boundary condition ! ------------------------------------------------------------------------------- - call ED_SunShadeFracs(this%fates(nc)%nsites, & + call FatesSunShadeFracs(this%fates(nc)%nsites, & this%fates(nc)%sites, & this%fates(nc)%bc_in, & this%fates(nc)%bc_out) @@ -2669,7 +2669,7 @@ subroutine wrap_canopy_radiation(this, bounds_clump, nc, & end do end do - call ED_Norman_Radiation(this%fates(nc)%nsites, & + call FatesNormalizedCanopyRadiation(this%fates(nc)%nsites, & this%fates(nc)%sites, & this%fates(nc)%bc_in, & this%fates(nc)%bc_out) @@ -2886,7 +2886,7 @@ subroutine wrap_update_hifrq_hist(this, bounds_clump, & this%fates(nc)%sites, & this%fates(nc)%bc_in, & dtime) - + end associate call t_stopf('fates_wrap_update_hifrq_hist') diff --git a/tools/site_and_regional/run_neon b/tools/site_and_regional/run_neon index ad930f50e3..ffc3be2af7 100755 --- a/tools/site_and_regional/run_neon +++ b/tools/site_and_regional/run_neon @@ -41,6 +41,7 @@ _CTSM_PYTHON = os.path.join( ) sys.path.insert(1, _CTSM_PYTHON) +# pylint: disable=import-error, wrong-import-position from ctsm.site_and_regional.run_neon import main if __name__ == "__main__":