Merge pull request #2 from sgfoote/master

Add optional fail span to climatology test

VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

conda-recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: ioos_qc_py2
-  version: "0.2.1"
+  version: "0.2.2"
 
 source:
   path: ../

docs/source/conf.py

@@ -82,9 +82,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = "0.2.1"
+version = "0.2.2"
 # The full version, including alpha/beta/rc tags.
-release = "0.2.1"
+release = "0.2.2"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

ioos_qc/__init__.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # coding=utf-8
-__version__ = "0.2.1"
+__version__ = "0.2.2"

ioos_qc/qartod.py

@@ -205,6 +205,7 @@ class ClimatologyConfig(object):
         tspan: 2-tuple range.
                 If period is defined, then this is a numeric range.
                 If period is not defined, then its a date range.
+        fspan: (optional) 2-tuple range of valid values. This is passed in as the fail_span to the gross_range test.
         vspan: 2-tuple range of valid values. This is passed in as the suspect_span to the gross_range test.
         zspan: (optional) Vertical (depth) range, in meters positive down
         period: (optional) The unit the tspan argument is in. Defaults to datetime object
@@ -218,6 +219,7 @@ class ClimatologyConfig(object):
     """
     mem = namedtuple('window', [
         'tspan',
+        'fspan',
         'vspan',
         'zspan',
         'period'
@@ -259,7 +261,7 @@ def values(self, tind, zind=None):
                     span = m.vspan
         return span
 
-    def add(self, tspan, vspan, zspan = None, period = None):
+    def add(self, tspan, vspan, fspan = None, zspan = None, period = None):
 
         assert isfixedlength(tspan, 2)
         # If period is defined, tspan is a numeric
@@ -275,6 +277,10 @@ def add(self, tspan, vspan, zspan = None, period = None):
         assert isfixedlength(vspan, 2)
         vspan = span(*sorted(vspan))
 
+        if fspan is not None:
+            assert isfixedlength(fspan, 2)
+            fspan = span(*sorted(fspan))
+
         if zspan is not None:
             assert isfixedlength(zspan, 2)
             zspan = span(*sorted(zspan))
@@ -289,14 +295,14 @@ def add(self, tspan, vspan, zspan = None, period = None):
         self._members.append(
             self.mem(
                 tspan,
+                fspan,
                 vspan,
                 zspan,
                 period
             )
         )
 
     def check(self, tinp, inp, zinp):
-
         # Start with everything as UNKNOWN (1)
         flag_arr = np.ma.empty(inp.size, dtype='uint8')
         flag_arr.fill(QartodFlags.UNKNOWN)
@@ -320,33 +326,39 @@ def check(self, tinp, inp, zinp):
                 tinp_copy = tinp
 
             # If a zspan is defined but we don't have z input (zinp), skip this member
-            # Note: `zinp.any()` can return `np.ma.masked` so we also check using isnan
-            if not isnan(m.zspan) and (not zinp.any() or isnan(zinp.any())):
+            # Note: `zinp.count()` can return `np.ma.masked` so we also check using isnan
+            if not isnan(m.zspan) and (not zinp.count() or isnan(zinp.any())):
                 continue
 
             # Indexes that align with the T
-            t_idx = (tinp_copy > m.tspan.minv) & (tinp_copy <= m.tspan.maxv)
+            t_idx = (tinp_copy >= m.tspan.minv) & (tinp_copy <= m.tspan.maxv)
 
             # Indexes that align with the Z
             if not isnan(m.zspan):
                 # Only test non-masked values between the min and max
-                z_idx = (~zinp.mask) & (zinp > m.zspan.minv) & (zinp <= m.zspan.maxv)
+                z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv)
             else:
-                # Only test the values with masked Z, ie values with no Z
-                z_idx = zinp.mask
+                # If there is no z data in the config, don't try to filter by depth!
+                # Set z_idx to all True to prevent filtering
+                z_idx = np.ones(inp.size, dtype=bool)
 
             # Combine the T and Z indexes
             values_idx = (t_idx & z_idx)
 
-            # Suspect data for this value span. Combined with the values_idx it
-            # represents the subset ofdata that should be suspect for this member.
-            # We split it into two indexes so we can also set all values outside of the
-            # suspect range to GOOD by taking the inverse of the suspect_idx
+            # Failed and suspect data for this value span. Combining fail_idx or
+            # suspect_idx with values_idx represents the subsets of data that should be
+            # fail and suspect respectively.
+            if not isnan(m.fspan):
+                fail_idx = (inp < m.fspan.minv) | (inp > m.fspan.maxv)
+            else:
+                fail_idx = np.zeros(inp.size, dtype=bool)
+
             suspect_idx = (inp < m.vspan.minv) | (inp > m.vspan.maxv)
 
             with np.errstate(invalid='ignore'):
-                flag_arr[(values_idx & suspect_idx)] = QartodFlags.SUSPECT
-                flag_arr[(values_idx & ~suspect_idx)] = QartodFlags.GOOD
+                flag_arr[(values_idx & fail_idx)] = QartodFlags.FAIL
+                flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT
+                flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD
 
         return flag_arr
 
@@ -362,7 +374,7 @@ def convert(config):
         return c
 
 
-def climatology_test(config, inp, tinp, zinp):
+def climatology_test(config, inp, tinp, zinp=None):
     """Checks that values are within reasonable range bounds and flags as SUSPECT.
 
     Data for which no ClimatologyConfig member exists is marked as UNKNOWN.
@@ -386,6 +398,7 @@ def climatology_test(config, inp, tinp, zinp):
     config = ClimatologyConfig.convert(config)
 
     tinp = mapdates(tinp)
+
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         inp = np.ma.masked_invalid(np.array(inp).astype(np.floating))

pytest.ini

@@ -1,6 +1,6 @@
 [pytest]
 
-addopts = -s -rxs -v
+addopts = -rxs -v
 
 flake8-max-line-length = 100
 flake8-ignore =

tests/test_qartod.py

@@ -444,6 +444,217 @@ def test_weekofyear_periods(self):
         self._run_test(cc)
 
 
+class QartodClimatologyInclusiveRangesTest(unittest.TestCase):
+    # Test that the various configuration spans (tspan, vspan, fspan, zspan) are
+    # inclusive of both endpoints.
+    def setUp(self):
+        self.cc = qartod.ClimatologyConfig()
+        self.cc.add(
+            tspan=(np.datetime64('2019-11-01'), np.datetime64('2020-02-04')),
+            fspan=(40, 70),
+            vspan=(50, 60),
+            zspan=(0, 10)
+        )
+
+    # test_inputs is a tuple of (time, value, depth) values representing science data from an instrument
+    # time should be an object compatible with pandas DateTimeIndex; value and depth should be intergers or floats
+    def _run_test(self, test_inputs, expected_result):
+        times, values, depths = zip(*test_inputs)
+        inputs = [
+            values,
+            np.asarray(values, dtype=np.floating),
+            dask_arr(np.asarray(values, dtype=np.floating))
+        ]
+
+        for i in inputs:
+            results = qartod.climatology_test(
+                config=self.cc,
+                tinp=times,
+                inp=i,
+                zinp=depths
+            )
+            npt.assert_array_equal(
+                results,
+                np.ma.array(expected_result)
+            )
+
+    def test_tspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2019-10-31'),
+                55,
+                5
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2019-11-01'),
+                55,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-02-04'),
+                55,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-02-05'),
+                55,
+                5
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                49,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                50,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                60,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                61,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                30,
+                5
+            )
+        ]
+        expected_result=[4]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                40,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                70,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                71,
+                5
+            )
+        ]
+        expected_result=[4]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                -1
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                0
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                10
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                11
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+
 class QartodClimatologyDepthTest(unittest.TestCase):
 
     def setUp(self):