Cellmethod tolerance

lib/iris/fileformats/netcdf/__init__.py

@@ -18,15 +18,14 @@
 # Note: *must* be done before importing from submodules, as they also use this !
 logger = iris.config.get_logger(__name__)
 
+from ._parse_cell_methods import UnknownCellMethodWarning, parse_cell_methods
 from .loader import DEBUG, NetCDFDataProxy, load_cubes
 from .saver import (
     CF_CONVENTIONS_VERSION,
     MESH_ELEMENTS,
     SPATIO_TEMPORAL_AXES,
     CFNameCoordMap,
     Saver,
-    UnknownCellMethodWarning,
-    parse_cell_methods,
     save,
 )
 

lib/iris/fileformats/netcdf/_parse_cell_methods.py

@@ -0,0 +1,219 @@
+# Copyright Iris contributors
+#
+# This file is part of Iris and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Helper routines specific to cell method parsing for netcdf-CF loading.
+
+"""
+import re
+from typing import List
+import warnings
+
+from iris.coords import CellMethod
+
+# Cell methods.
+_CM_KNOWN_METHODS = [
+    "point",
+    "sum",
+    "mean",
+    "maximum",
+    "minimum",
+    "mid_range",
+    "standard_deviation",
+    "variance",
+    "mode",
+    "median",
+]
+
+_CM_COMMENT = "comment"
+_CM_EXTRA = "extra"
+_CM_INTERVAL = "interval"
+_CM_METHOD = "method"
+_CM_NAME = "name"
+_CM_PARSE_NAME = re.compile(r"([\w_]+\s*?:\s+)+")
+_CM_PARSE = re.compile(
+    r"""
+                           (?P<name>([\w_]+\s*?:\s+)+)
+                           (?P<method>[\w_\s]+(?![\w_]*\s*?:))\s*
+                           (?:
+                               \(\s*
+                               (?P<extra>.+)
+                               \)\s*
+                           )?
+                       """,
+    re.VERBOSE,
+)
+
+
+class UnknownCellMethodWarning(Warning):
+    pass
+
+
+def _split_cell_methods(nc_cell_methods: str) -> List[re.Match]:
+    """
+    Split a CF cell_methods attribute string into a list of zero or more cell
+    methods, each of which is then parsed with a regex to return a list of match
+    objects.
+
+    Args:
+
+    * nc_cell_methods: The value of the cell methods attribute to be split.
+
+    Returns:
+
+    * nc_cell_methods_matches: A list of the re.Match objects associated with
+      each parsed cell method
+
+    Splitting is done based on words followed by colons outside of any brackets.
+    Validation of anything other than being laid out in the expected format is
+    left to the calling function.
+    """
+
+    # Find name candidates
+    name_start_inds = []
+    for m in _CM_PARSE_NAME.finditer(nc_cell_methods):
+        name_start_inds.append(m.start())
+
+    # Remove those that fall inside brackets
+    bracket_depth = 0
+    for ind, cha in enumerate(nc_cell_methods):
+        if cha == "(":
+            bracket_depth += 1
+        elif cha == ")":
+            bracket_depth -= 1
+            if bracket_depth < 0:
+                msg = (
+                    "Cell methods may be incorrectly parsed due to mismatched "
+                    "brackets"
+                )
+                warnings.warn(msg, UserWarning, stacklevel=2)
+        if bracket_depth > 0 and ind in name_start_inds:
+            name_start_inds.remove(ind)
+
+    # List tuples of indices of starts and ends of the cell methods in the string
+    name_start_inds.append(len(nc_cell_methods))
+    method_indices = list(zip(name_start_inds[:-1], name_start_inds[1:]))
+
+    # Index the string and match against each substring
+    nc_cell_methods_matches = []
+    for start_ind, end_ind in method_indices:
+        nc_cell_method_str = nc_cell_methods[start_ind:end_ind]
+        nc_cell_method_match = _CM_PARSE.match(nc_cell_method_str.strip())
+        if not nc_cell_method_match:
+            msg = (
+                f"Failed to fully parse cell method string: {nc_cell_methods}"
+            )
+            warnings.warn(msg, UserWarning, stacklevel=2)
+            continue
+        nc_cell_methods_matches.append(nc_cell_method_match)
+
+    return nc_cell_methods_matches
+
+
+def parse_cell_methods(nc_cell_methods):
+    """
+    Parse a CF cell_methods attribute string into a tuple of zero or
+    more CellMethod instances.
+
+    Args:
+
+    * nc_cell_methods (str):
+        The value of the cell methods attribute to be parsed.
+
+    Returns:
+
+    * cell_methods
+        An iterable of :class:`iris.coords.CellMethod`.
+
+    Multiple coordinates, intervals and comments are supported.
+    If a method has a non-standard name a warning will be issued, but the
+    results are not affected.
+
+    """
+
+    cell_methods = []
+    if nc_cell_methods is not None:
+        splits = _split_cell_methods(nc_cell_methods)
+        if not splits:
+            msg = (
+                f"NetCDF variable cell_methods of {nc_cell_methods!r} "
+                "contains no valid cell methods."
+            )
+            warnings.warn(msg, UserWarning)
+        for m in splits:
+            d = m.groupdict()
+            method = d[_CM_METHOD]
+            method = method.strip()
+            # Check validity of method, allowing for multi-part methods
+            # e.g. mean over years.
+            method_words = method.split()
+            if method_words[0].lower() not in _CM_KNOWN_METHODS:
+                msg = "NetCDF variable contains unknown cell method {!r}"
+                warnings.warn(
+                    msg.format("{}".format(method_words[0])),
+                    UnknownCellMethodWarning,
+                )
+            d[_CM_METHOD] = method
+            name = d[_CM_NAME]
+            name = name.replace(" ", "")
+            name = name.rstrip(":")
+            d[_CM_NAME] = tuple([n for n in name.split(":")])
+            interval = []
+            comment = []
+            if d[_CM_EXTRA] is not None:
+                #
+                # tokenise the key words and field colon marker
+                #
+                d[_CM_EXTRA] = d[_CM_EXTRA].replace(
+                    "comment:", "<<comment>><<:>>"
+                )
+                d[_CM_EXTRA] = d[_CM_EXTRA].replace(
+                    "interval:", "<<interval>><<:>>"
+                )
+                d[_CM_EXTRA] = d[_CM_EXTRA].split("<<:>>")
+                if len(d[_CM_EXTRA]) == 1:
+                    comment.extend(d[_CM_EXTRA])
+                else:
+                    next_field_type = comment
+                    for field in d[_CM_EXTRA]:
+                        field_type = next_field_type
+                        index = field.rfind("<<interval>>")
+                        if index == 0:
+                            next_field_type = interval
+                            continue
+                        elif index > 0:
+                            next_field_type = interval
+                        else:
+                            index = field.rfind("<<comment>>")
+                            if index == 0:
+                                next_field_type = comment
+                                continue
+                            elif index > 0:
+                                next_field_type = comment
+                        if index != -1:
+                            field = field[:index]
+                        field_type.append(field.strip())
+            #
+            # cater for a shared interval over multiple axes
+            #
+            if len(interval):
+                if len(d[_CM_NAME]) != len(interval) and len(interval) == 1:
+                    interval = interval * len(d[_CM_NAME])
+            #
+            # cater for a shared comment over multiple axes
+            #
+            if len(comment):
+                if len(d[_CM_NAME]) != len(comment) and len(comment) == 1:
+                    comment = comment * len(d[_CM_NAME])
+            d[_CM_INTERVAL] = tuple(interval)
+            d[_CM_COMMENT] = tuple(comment)
+            cell_method = CellMethod(
+                d[_CM_METHOD],
+                coords=d[_CM_NAME],
+                intervals=d[_CM_INTERVAL],
+                comments=d[_CM_COMMENT],
+            )
+            cell_methods.append(cell_method)
+    return tuple(cell_methods)