ctsm5.3.021: Standardize time metadata (we will mark this as the release tag for ctsm5.3)

.git-blame-ignore-revs

@@ -1,4 +1,5 @@
 # Ran python directory through black python formatter
+d89c86e7776bbf7451860b60038b4725fe7a0560
 b429b63824e09f82e95d2982f14311cbbd8e4a37
 d229b5c6689efc4c2a6cef077515c4ccd5c18ff6
 4cd83cb3ee6d85eb909403487abf5eeaf4d98911

.gitmodules

@@ -52,7 +52,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/RTM
 [submodule "mosart"]
 path = components/mosart
 url = https://github.com/ESCOMP/MOSART
-fxtag = mosart1.1.07
+fxtag = mosart1.1.08
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/MOSART

cime_config/SystemTests/rxcropmaturity.py

@@ -106,7 +106,7 @@ def __init__(self, case):
         # Which conda environment should we use?
         self._get_conda_env()
 
-    def _run_phase(self, skip_gen=False):
+    def _run_phase(self, skip_gen=False, h1_inst=False):
         # Modeling this after the SSP test, we create a clone to be the case whose outputs we don't
         # want to be saved as baseline.
 
@@ -129,7 +129,7 @@ def _run_phase(self, skip_gen=False):
         self._set_active_case(case_gddgen)
 
         # Set up stuff that applies to both tests
-        self._setup_all()
+        self._setup_all(h1_inst)
 
         # Add stuff specific to GDD-Generating run
         logger.info("RXCROPMATURITY log:  modify user_nl files: generate GDDs")
@@ -196,7 +196,7 @@ def _run_phase(self, skip_gen=False):
         self._set_active_case(case_rxboth)
 
         # Set up stuff that applies to both tests
-        self._setup_all()
+        self._setup_all(h1_inst)
 
         # Add stuff specific to Prescribed Calendars run
         logger.info("RXCROPMATURITY log:  modify user_nl files: Prescribed Calendars")
@@ -264,7 +264,7 @@ def _get_rx_dates(self):
             logger.error(error_message)
             raise RuntimeError(error_message)
 
-    def _setup_all(self):
+    def _setup_all(self, h1_inst):
         logger.info("RXCROPMATURITY log:  _setup_all start")
 
         # Get some info
@@ -274,7 +274,7 @@ def _setup_all(self):
 
         # Set sowing dates file (and other crop calendar settings) for all runs
         logger.info("RXCROPMATURITY log:  modify user_nl files: all tests")
-        self._modify_user_nl_allruns()
+        self._modify_user_nl_allruns(h1_inst)
         logger.info("RXCROPMATURITY log:  _setup_all done")
 
     # Make a surface dataset that has every crop in every gridcell
@@ -399,7 +399,7 @@ def _run_check_rxboth_run(self, skip_gen):
             tool_path,
         )
 
-    def _modify_user_nl_allruns(self):
+    def _modify_user_nl_allruns(self, h1_inst):
         nl_additions = [
             "cropcals_rx = .true.",
             "cropcals_rx_adapt = .false.",
@@ -417,6 +417,8 @@ def _modify_user_nl_allruns(self):
             "hist_type1d_pertape(2) = 'PFTS'",
             "hist_dov2xy(2) = .false.",
         ]
+        if h1_inst:
+            nl_additions.append("hist_avgflag_pertape(2) = 'I'")
         self._append_to_user_nl_clm(nl_additions)
 
     def _run_generate_gdds(self, case_gddgen):

cime_config/SystemTests/rxcropmaturityinst.py

@@ -0,0 +1,6 @@
+from rxcropmaturity import RXCROPMATURITYSHARED
+
+
+class RXCROPMATURITYINST(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase(h1_inst=True)

cime_config/SystemTests/rxcropmaturityskipgeninst.py

@@ -0,0 +1,6 @@
+from rxcropmaturity import RXCROPMATURITYSHARED
+
+
+class RXCROPMATURITYSKIPGENINST(RXCROPMATURITYSHARED):
+    def run_phase(self):
+        self._run_phase(skip_gen=True, h1_inst=True)

cime_config/config_tests.xml

@@ -145,6 +145,16 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="RXCROPMATURITYINST">
+    <DESC>As RXCROPMATURITY but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
   <test NAME="RXCROPMATURITYSKIPGEN">
     <DESC>As RXCROPMATURITY but don't actually generate GDDs. Allows short testing with existing GDD inputs.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
@@ -155,6 +165,16 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="RXCROPMATURITYSKIPGENINST">
+    <DESC>As RXCROPMATURITYSKIPGEN but ensure instantaneous h1. Can be removed once instantaneous and other variables are on separate files.</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
 <!--
 SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)

cime_config/testdefs/testlist_clm.xml

@@ -3649,6 +3649,17 @@
     </options>
   </test>
 
+  <test name="RXCROPMATURITYINST_Lm61" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+      <machine name="derecho" compiler="intel" category="crop_calendars"/>
+    </machines>
+    <options>
+      <option name="wallclock">6:00:00</option>
+      <option name="comment">As RXCROPMATURITY, but ensure that h1 file is instantaneous. Can be removed once instantaneous and other variables are separated onto separate files.</option>
+    </options>
+  </test>
+
   <test name="RXCROPMATURITYSKIPGEN_Ld1097" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>
@@ -3661,6 +3672,17 @@
     </options>
   </test>
 
+  <test name="RXCROPMATURITYSKIPGENINST_Ld1097" grid="f10_f10_mg37" compset="IHistClm60BgcCrop"  testmods="clm/cropMonthOutput">
+    <machines>
+      <machine name="derecho" compiler="intel" category="rxcropmaturity"/>
+      <machine name="derecho" compiler="intel" category="crop_calendars"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:45:00</option>
+      <option name="comment">As RXCROPMATURITYSKIPGEN, but ensure that h1 file is instantaneous. Can be removed once instantaneous and other variables are separated onto separate files.</option>
+    </options>
+  </test>
+
   <test name="ERP_D_P64x2_Ld10" grid="f10_f10_mg37" compset="I2000Clm60Bgc" testmods="clm/Hillslope">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>

doc/ChangeLog

@@ -1,4 +1,97 @@
 ===============================================================
+Tag name: ctsm5.3.021
+Originator(s): samrabin (Sam Rabin, UCAR/TSS)
+Date: Fri 17 Jan 2025 12:42:24 PM MST
+One-line Summary: Standardize time metadata
+
+Purpose and description of changes
+----------------------------------
+ Standardizes a dimension name of output variable time_bounds from hist_interval to nbnd.
+ Standardizes attributes for time_bounds, mcdate, mcsec, mdcur, and mscur.
+
+Contributors
+------------
+ Also Bill Sacks and Erik Kluzek.
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm6_0
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed
+----------
+List of CTSM issues fixed (include CTSM Issue # and description) [one per line]:
+ Resolves #1693
+ Fixes #2923
+ Resolves https://github.com/ESCOMP/MOSART/pull/66
+ Resolves https://github.com/ESCOMP/RTM/pull/35
+
+Notes of particular relevance for developers:
+---------------------------------------------
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+ Same or similar changes needed in all three: clm, rtm, mosart.
+
+Testing summary:
+----------------
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  build-namelist tests (if CLMBuildNamelist.pm has changed):
+
+    derecho -
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    derecho -
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    derecho -----
+    izumi -------
+
+  fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates-<FATES TAG>-<CTSM TAG>)
+    derecho -----
+    izumi -------
+
+  any other testing (give details below):
+
+  ctsm_sci
+     derecho ----
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: No
+ Just new metadata in history.
+
+Other details
+-------------
+List any git submodules updated (cime, rtm, mosart, cism, fates, etc.):
+ mosart1.1.08
+ rtm1_0_85
+
+Pull Requests that document the changes (include PR ids):
+ https://github.com/ESCOMP/ctsm/pull/2052
+ https://github.com/ESCOMP/MOSART/pull/66
+ https://github.com/ESCOMP/RTM/pull/35
+
+===============================================================
+===============================================================
 Tag name: ctsm5.3.020
 Originator(s): samrabin (Sam Rabin, UCAR/TSS, [email protected])
 Date: Fri Jan 17 12:21:24 MST 2025
@@ -65,7 +158,7 @@ Purpose and description of changes
 
 Contributors
 ------------
- Bill Sacks. In the final steps, also slevis.
+ Also Bill Sacks and, in the final steps, slevis.
 
 Significant changes to scientifically-supported configurations
 --------------------------------------------------------------

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+       ctsm5.3.021 samrabin 01/17/2025 Standardize time metadata
        ctsm5.3.020 samrabin 01/17/2025 Merge b4b-dev
        ctsm5.3.019   olyson 01/14/2025 Stop running 0th time step
        ctsm5.3.018   slevis 01/10/2025 Change history time to be the middle of the time bounds

python/ctsm/crop_calendars/convert_axis_time2gs.py

@@ -5,6 +5,7 @@
 import sys
 import numpy as np
 import xarray as xr
+from ctsm.crop_calendars.cropcal_utils import get_integer_years
 
 try:
     import pandas as pd
@@ -85,7 +86,7 @@ def set_up_ds_with_gs_axis(ds_in):
         if not any(x in ["mxsowings", "mxharvests"] for x in ds_in[var].dims):
             data_vars[var] = ds_in[var]
     # Set up the new dataset
-    gs_years = [t.year - 1 for t in ds_in.time.values[:-1]]
+    gs_years = get_integer_years(ds_in)[:-1]
     coords = ds_in.coords
     coords["gs"] = gs_years
     ds_out = xr.Dataset(data_vars=data_vars, coords=coords, attrs=ds_in.attrs)

python/ctsm/crop_calendars/cropcal_figs_module.py

@@ -13,6 +13,7 @@
 from matplotlib import cm
 import matplotlib.collections as mplcol
 
+# pylint: disable=abstract-class-instantiated
 
 # Colormaps (maps)
 cropcal_colors = {

python/ctsm/crop_calendars/cropcal_module.py

@@ -20,24 +20,14 @@ def check_and_trim_years(year_1, year_n, ds_in):
     """
     After importing a file, restrict it to years of interest.
     """
-    ### In annual outputs, file with name Y is actually results from year Y-1.
-    ### Note that time values refer to when it was SAVED. So 1981-01-01 is for year 1980.
-
-    def get_year_from_cftime(cftime_date):
-        # Subtract 1 because the date for annual files is when it was SAVED
-        return cftime_date.year - 1
 
     # Check that all desired years are included
-    if get_year_from_cftime(ds_in.time.values[0]) > year_1:
-        raise RuntimeError(
-            f"Requested year_1 is {year_1} but first year in outputs is "
-            + f"{get_year_from_cftime(ds_in.time.values[0])}"
-        )
-    if get_year_from_cftime(ds_in.time.values[-1]) < year_1:
-        raise RuntimeError(
-            f"Requested year_n is {year_n} but last year in outputs is "
-            + f"{get_year_from_cftime(ds_in.time.values[-1])}"
-        )
+    year = utils.get_timestep_year(ds_in, ds_in.time.values[0])
+    if year > year_1:
+        raise RuntimeError(f"Requested year_1 is {year_1} but first year in outputs is {year}")
+    year = utils.get_timestep_year(ds_in, ds_in.time.values[-1])
+    if year < year_1:
+        raise RuntimeError(f"Requested year_n is {year_n} but last year in outputs is {year}")
 
     # Remove years outside range of interest
     ### Include an extra year at the end to finish out final seasons.

python/ctsm/crop_calendars/cropcal_utils.py

@@ -430,3 +430,45 @@ def make_lon_increasing(xr_obj):
         raise RuntimeError("Unable to rearrange longitude axis so it's monotonically increasing")
 
     return xr_obj.roll(lon=shift, roll_coords=True)
+
+
+def is_inst_file(dsa):
+    """
+    Check whether Dataset or DataArray has time data from an "instantaneous file"
+    """
+    return "at end of" in dsa["time"].attrs["long_name"]
+
+
+def get_beg_inst_timestep_year(timestep):
+    """
+    Get year associated with the BEGINNING of a timestep in an
+    instantaneous file
+    """
+    year = timestep.year
+
+    is_jan1 = timestep.dayofyr == 1
+    is_midnight = timestep.hour == timestep.minute == timestep.second == 0
+    if is_jan1 and is_midnight:
+        year -= 1
+
+    return year
+
+
+def get_timestep_year(dsa, timestep):
+    """
+    Get the year associated with a timestep, with different handling
+    depending on whether the file is instantaneous
+    """
+    if is_inst_file(dsa):
+        year = get_beg_inst_timestep_year(timestep)
+    else:
+        year = timestep.year
+    return year
+
+
+def get_integer_years(dsa):
+    """
+    Convert time axis to numpy array of integer years
+    """
+    out_array = [get_timestep_year(dsa, t) for t in dsa["time"].values]
+    return out_array

python/ctsm/crop_calendars/generate_gdds_functions.py

@@ -553,8 +553,8 @@ def import_and_process_1yr(
     clm_gdd_var = "GDDACCUM"
     my_vars = [clm_gdd_var, "GDDHARV"]
     patterns = [f"*h2.{this_year-1}-01*.nc", f"*h2.{this_year-1}-01*.nc.base"]
-    for p in patterns:
-        pattern = os.path.join(indir, p)
+    for pat in patterns:
+        pattern = os.path.join(indir, pat)
         h2_files = glob.glob(pattern)
         if h2_files:
             break