Merge pull request ESCOMP#2604 from slevis-lmwg/merge-b4bdev-20240620

Merge b4bdev 20240620

.git-blame-ignore-revs

@@ -46,3 +46,4 @@ aa04d1f7d86cc2503b98b7e2b2d84dbfff6c316b
 6c6f57e948bfa31e60b383536cc21663fedb8b70
 9660667b1267dcd4150889f5f39db540158be74a
 665cf86102e09b4c4c5a140700676dca23bc55a9
+045d90f1d80f713eb3ae0ac58f6c2352937f1eb0

cime_config/SystemTests/rxcropmaturity.py

@@ -20,17 +20,25 @@
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files
+from CIME.case import Case
 import shutil, glob
 
 logger = logging.getLogger(__name__)
 
 
-class RXCROPMATURITY(SystemTestsCommon):
+class RXCROPMATURITYSHARED(SystemTestsCommon):
     def __init__(self, case):
         # initialize an object interface to the SMS system test
         SystemTestsCommon.__init__(self, case)
 
-        # Ensure run length is at least 5 years. Minimum to produce one complete growing season (i.e., two complete calendar years) actually 4 years, but that only gets you 1 season usable for GDD generation, so you can't check for season-to-season consistency.
+        # Is this a real RXCROPMATURITY test or not?
+        casebaseid = self._case.get_value("CASEBASEID")
+        full_test = "RXCROPMATURITY_" in casebaseid
+        skipgen_test = "RXCROPMATURITYSKIPGEN_" in casebaseid
+
+        # Ensure run length is at least 5 years. Minimum to produce one complete growing season
+        # (i.e., two complete calendar years) actually 4 years, but that only gets you 1 season
+        # usable for GDD generation, so you can't check for season-to-season consistency.
         stop_n = self._case.get_value("STOP_N")
         stop_option = self._case.get_value("STOP_OPTION")
         stop_n_orig = stop_n
@@ -56,11 +64,20 @@ def __init__(self, case):
                 f"STOP_OPTION ({stop_option_orig}) must be nsecond(s), nminute(s), "
                 + "nhour(s), nday(s), nmonth(s), or nyear(s)"
             )
-        elif stop_n < 5:
+        elif full_test and stop_n < 5:
             error_message = (
                 "RXCROPMATURITY must be run for at least 5 years; you requested "
                 + f"{stop_n_orig} {stop_option_orig[1:]}"
             )
+        elif skipgen_test and stop_n < 3:
+            # First year is discarded because crops are already in the ground at restart, and those
+            # aren't affected by the new crop calendar inputs. The second year is useable, but we
+            # need a third year so that all crops planted in the second year have a chance to
+            # finish.
+            error_message = (
+                "RXCROPMATURITYSKIPGEN (both-forced part) must be run for at least 3 years; you requested "
+                + f"{stop_n_orig} {stop_option_orig[1:]}"
+            )
         if error_message is not None:
             logger.error(error_message)
             raise RuntimeError(error_message)
@@ -69,7 +86,6 @@ def __init__(self, case):
         self._run_Nyears = int(stop_n)
 
         # Only allow RXCROPMATURITY to be called with test cropMonthOutput
-        casebaseid = self._case.get_value("CASEBASEID")
         if casebaseid.split("-")[-1] != "cropMonthOutput":
             error_message = (
                 "Only call RXCROPMATURITY with test cropMonthOutput "
@@ -81,10 +97,16 @@ def __init__(self, case):
         # Get files with prescribed sowing and harvest dates
         self._get_rx_dates()
 
+        # Get cultivar maturity requirement file to fall back on if not generating it here
+        self._gdds_file = None
+        self._fallback_gdds_file = os.path.join(
+            os.path.dirname(self._sdatefile), "gdds_20230829_161011.nc"
+        )
+
         # Which conda environment should we use?
         self._get_conda_env()
 
-    def run_phase(self):
+    def _run_phase(self, skip_gen=False):
         # Modeling this after the SSP test, we create a clone to be the case whose outputs we don't
         # want to be saved as baseline.
 
@@ -133,6 +155,12 @@ def run_phase(self):
             # Download files from the server, if needed
             case_gddgen.check_all_input_data()
 
+            # Copy needed file from original to gddgen directory
+            shutil.copyfile(
+                os.path.join(caseroot, ".env_mach_specific.sh"),
+                os.path.join(self._path_gddgen, ".env_mach_specific.sh"),
+            )
+
             # Make custom version of surface file
             logger.info("RXCROPMATURITY log:  run fsurdat_modifier")
             self._run_fsurdat_modifier()
@@ -146,9 +174,19 @@ def run_phase(self):
         # "No history files expected, set suffix=None to avoid compare error"
         # We *do* expect history files here, but anyway. This works.
         self._skip_pnl = False
-        self.run_indv(suffix=None, st_archive=True)
 
-        self._run_generate_gdds(case_gddgen)
+        # If not generating GDDs, only run a few days of this.
+        if skip_gen:
+            with Case(self._path_gddgen, read_only=False) as case:
+                case.set_value("STOP_N", 5)
+                case.set_value("STOP_OPTION", "ndays")
+
+        self.run_indv(suffix=None, st_archive=True)
+        if skip_gen:
+            # Interpolate an existing GDD file. Needed to check obedience to GDD inputs.
+            self._run_interpolate_gdds()
+        else:
+            self._run_generate_gdds(case_gddgen)
 
         # -------------------------------------------------------------------
         # (3) Set up and perform Prescribed Calendars run
@@ -174,7 +212,7 @@ def run_phase(self):
         # (4) Check Prescribed Calendars run
         # -------------------------------------------------------------------
         logger.info("RXCROPMATURITY log:  output check: Prescribed Calendars")
-        self._run_check_rxboth_run()
+        self._run_check_rxboth_run(skip_gen)
 
     # Get sowing and harvest dates for this resolution.
     def _get_rx_dates(self):
@@ -331,11 +369,16 @@ def _create_config_file_evenlysplitcrop(self):
             cfg_out.write("PCT_OCEAN   = 0.0\n")
             cfg_out.write("PCT_URBAN   = 0.0 0.0 0.0\n")
 
-    def _run_check_rxboth_run(self):
+    def _run_check_rxboth_run(self, skip_gen):
 
         output_dir = os.path.join(self._get_caseroot(), "run")
-        first_usable_year = self._run_startyear + 2
-        last_usable_year = self._run_startyear + self._run_Nyears - 2
+
+        if skip_gen:
+            first_usable_year = self._run_startyear + 1
+            last_usable_year = first_usable_year
+        else:
+            first_usable_year = self._run_startyear + 2
+            last_usable_year = self._run_startyear + self._run_Nyears - 2
 
         tool_path = os.path.join(
             self._ctsm_root, "python", "ctsm", "crop_calendars", "check_rxboth_run.py"
@@ -386,6 +429,7 @@ def _run_generate_gdds(self, case_gddgen):
         hdates_file = self._hdatefile
 
         # It'd be much nicer to call generate_gdds.main(), but I can't import generate_gdds.
+        # See https://github.com/ESCOMP/CTSM/issues/2603
         tool_path = os.path.join(
             self._ctsm_root, "python", "ctsm", "crop_calendars", "generate_gdds.py"
         )
@@ -416,6 +460,31 @@ def _run_generate_gdds(self, case_gddgen):
             raise RuntimeError(error_message)
         self._gdds_file = generated_gdd_files[0]
 
+    def _run_interpolate_gdds(self):
+        # File where interpolated GDDs should be saved
+        self._gdds_file = os.path.join(self._get_caseroot(), "interpolated_gdds.nc")
+
+        # It'd be much nicer to call interpolate_gdds.main(), but I can't import interpolate_gdds.
+        # See https://github.com/ESCOMP/CTSM/issues/2603
+        tool_path = os.path.join(
+            self._ctsm_root, "python", "ctsm", "crop_calendars", "interpolate_gdds.py"
+        )
+        command = " ".join(
+            [
+                f"python3 {tool_path}",
+                f"--input-file {self._fallback_gdds_file}",
+                f"--target-file {self._sdatefile}",
+                f"--output-file {self._gdds_file}",
+                "--overwrite",
+            ]
+        )
+        stu.run_python_script(
+            self._get_caseroot(),
+            self._this_conda_env,
+            command,
+            tool_path,
+        )
+
     def _get_conda_env(self):
         conda_setup_commands = stu.cmds_to_setup_conda(self._get_caseroot())
 
@@ -442,3 +511,8 @@ def _get_flanduse_timeseries_in(self, case):
                 if flanduse_timeseries_in:
                     self._flanduse_timeseries_in = flanduse_timeseries_in.group(1)
                     break
+
+
+class RXCROPMATURITY(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase()

cime_config/SystemTests/rxcropmaturityskipgen.py

@@ -0,0 +1,6 @@
+from rxcropmaturity import RXCROPMATURITYSHARED
+
+
+class RXCROPMATURITYSKIPGEN(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase(skip_gen=True)

cime_config/config_tests.xml

@@ -133,6 +133,16 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="RXCROPMATURITYSKIPGEN">
+    <DESC>As RXCROPMATURITY but don't actually generate GDDs. Allows short testing with existing GDD inputs.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
 <!--
 SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)

cime_config/testdefs/ExpectedTestFails.xml

@@ -29,13 +29,6 @@
 
 
   <!-- ctsm_sci test suite failures -->
-  <test name="RXCROPMATURITY_Lm61.f09_g17.IHistClm50BgcCrop.derecho_intel.clm-cropMonthOutput">
-    <phase name="RUN">
-      <status>FAIL</status>
-      <issue>#2460</issue>
-    </phase>
-  </test>
-
   <test name="SMS_Ld12_Mmpi-serial.1x1_urbanc_alpha.I1PtClm60SpRs.derecho_intel.clm-output_sp_highfreq">
     <phase name="RUN">
       <status>FAIL</status>

cime_config/testdefs/testlist_clm.xml

@@ -10,6 +10,7 @@
         prebeta: Run before CESM beta tags (more extensive, but should have tests outside of prealpha) (subset of aux_clm tests)
         aux_cime_baselines: CESM cime baselines (subset of aux_clm tests)
         hillslope: Experimental test list used for the hillslope option of the model
+        rxcropmaturity: Short tests to be run during development related to prescribed crop calendars
 -->
 <testlist version="2.0">
   <test name="ERI_D_Ld9" grid="f10_f10_mg37" compset="I1850Clm60Bgc" testmods="clm/default">
@@ -3021,6 +3022,27 @@
     </options>
   </test>
 
+  <test name="RXCROPMATURITY_Lm61" grid="f10_f10_mg37" compset="IHistClm50BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+    </machines>
+    <options>
+      <option name="wallclock">6:00:00</option>
+      <option name="comment">This test is designed to test the ability to prescribe crop sowing dates and maturity requirements. It first performs a GDD-generating run, then calls Python code to generate the maturity requirement file. This is then used in a sowing+maturity forced run, which finally is tested to ensure correct behavior.</option>
+    </options>
+  </test>
+
+  <test name="RXCROPMATURITYSKIPGEN_Ld1097" grid="f10_f10_mg37" compset="IHistClm50BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:45:00</option>
+      <option name="comment">This test is designed to test the ability to prescribe crop sowing dates and maturity requirements. Unlike the RXCROPMATURITY test, however, it does not generate its own maturity requirements, instead interpolating a pre-existing maturity requirement file. (This allows the "GDD-generating" run to be mostly skipped.) This is then used in a sowing+maturity forced run, which finally is tested to ensure correct behavior.</option>
+    </options>
+  </test>
+
   <test name="ERP_D_P64x2_Ld10" grid="f10_f10_mg37" compset="I2000Clm60Bgc" testmods="clm/Hillslope">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>

doc/ChangeLog

@@ -1,4 +1,84 @@
 ===============================================================
+Tag name: tmp-240620.n01.ctsm5.2.007
+Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310)
+Date: Fri 21 Jun 2024 09:42:26 AM MDT
+One-line Summary: Merge b4b-dev
+
+Purpose and description of changes
+----------------------------------
+
+ Fix long names #2512
+ Dust emissions moved to Object Oriented design #2552
+ Fix RXCROPMATURITY test #2599
+
+ NB: This is the first temporary tag (n01, see full tag name above) in
+ a series of temporary tags while we wait for the completion of the
+ beta01 cesm tag.
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm6_0
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+Bugs fixed
+----------
+List of CTSM issues fixed (include CTSM Issue # and description) [one per line]:
+ Fixes #2512
+ Fixes #2552
+ Fixes #2599
+
+Notes of particular relevance for developers:
+---------------------------------------------
+Changes to tests or testing:
+ Fix RXCROPMATURITY test #2599
+
+Testing summary:
+----------------
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  build-namelist tests (if CLMBuildNamelist.pm has changed):
+
+    derecho - PASS
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    derecho - PASS
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    derecho ----- OK
+    izumi ------- OK
+
+  any other testing (give details below):
+
+  ctsm_sci
+     derecho ----
+
+Answer changes
+--------------
+Changes answers relative to baseline: No
+
+Other details
+-------------
+Pull Requests that document the changes (include PR ids):
+ https://github.com/ESCOMP/ctsm/pull/2604
+
+===============================================================
+===============================================================
 Tag name: ctsm5.2.007
 Originator(s): jedwards4b (Jim Edwards, UCAR/CGD) slevis (Sam Levis, UCAR/CGD)
 Date: Fri 31 May 2024 13:49:29 AM MDT

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+tmp-240620.n01.cts   slevis 06/21/2024 Merge b4b-dev (full tag name is tmp-240620.n01.ctsm5.2.007)
        ctsm5.2.007 multiple 05/31/2024 Rm manage_externals and update documentation accordingly
        ctsm5.2.006   slevis 05/28/2024 Update externals to cesm2_3_beta17, remove mct, retire /test/tools
        ctsm5.2.005     erik 05/13/2024 Fix clm6_0 defaults and CESM testing issues, add tests to detect these problems

python/ctsm/crop_calendars/check_constant_vars.py

@@ -382,4 +382,4 @@ def check_constant_vars(
         raise RuntimeError("Stopping due to failed check_constant_vars().")
 
     bad_patches = np.unique(bad_patches)
-    return [int(p) for p in bad_patches]
+    return [int(p) for p in bad_patches], any_bad