Merge pull request #1 from sgfoote/qartod

Porting code back to python 2.7

README.rst

@@ -6,13 +6,12 @@ IOOS QC
    :target: https://anaconda.org/conda-forge/ioos_qc
    :alt: conda_forge_package
 
-.. image:: https://travis-ci.org/ioos/ioos_qc.svg?branch=master
-   :target: https://travis-ci.org/ioos/ioos_qc
-   :alt: build_status
+This is a backport of the ioos_qc package to work with python 2.7.
+The original package description follows:
 
 Collection of utilities, scripts and tests to assist in automated
 quality assurance and quality control for oceanographic datasets and
 observing systems.
 
-`Code <https://github.com/ioos/ioos_qc>`_  |  `Issues <https://github.com/ioos/ioos_qc/issues>`_  |  `Documentation <https://ioos.github.io/ioos_qc/>`_
+`Code <https://github.com/oceanobservatories/ioos_qc>`_  |  `Issues <https://github.com/oceanobservatories/ioos_qc/issues>`_  |  `Documentation <https://ioos.github.io/ioos_qc/>`_
 

conda-recipe/meta.yaml

@@ -1,5 +1,5 @@
 package:
-  name: ioos_qc
+  name: ioos_qc_py2
   version: "0.2.1"
 
 source:
@@ -15,6 +15,8 @@ requirements:
     - python
     - pip
   run:
+    - funcsigs
+    - pathlib
     - python
     - geojson
     - netCDF4

docs/requirements.txt

@@ -1,5 +1,3 @@
-doctr
 sphinx
-sphinx-autodoc-typehints
 nbsphinx
-ipykernel
+ipykernel

docs/source/conf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 #
 # ioos_qc documentation build configuration file, created by
@@ -40,7 +40,6 @@
     'sphinx.ext.todo',
     'sphinx.ext.mathjax',
     'sphinx.ext.viewcode',
-    'sphinx_autodoc_typehints',
     "nbsphinx",
 ]
 

docs/source/examples/QartodTestExample_WaterLevel.ipynb

@@ -211,21 +211,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
   }
  },
  "nbformat": 4,

docs/source/examples/Qartod_Single_Test_Example.ipynb

@@ -295,21 +295,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 2",
    "language": "python",
-   "name": "python3"
+   "name": "python2"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 2
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
   }
  },
  "nbformat": 4,

docs/source/installation.rst

@@ -11,8 +11,8 @@ Development and Testing
 
 Create a conda environment::
 
-    conda create -n ioosqc37 python=3.7
-    conda activate ioosqc37
+    conda create -n ioosqc27 python=2.7
+    conda activate ioosqc27
     conda install --file requirements.txt --file tests/requirements.txt
 
 Run tests::
@@ -22,8 +22,8 @@ Run tests::
 
 Build docs::
 
-    conda create -y -n ioosqc37_docs python=3.7
-    conda activate ioosqc37_docs
+    conda create -y -n ioosqc27_docs python=2.7
+    conda activate ioosqc27_docs
     conda install -y --file requirements.txt \
         --file tests/requirements.txt \
         --file docs/requirements.txt \

ioos_qc/config.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 import simplejson as json
 from copy import deepcopy
-from inspect import signature
+from funcsigs import signature
 from collections import OrderedDict
 from importlib import import_module
 
@@ -57,7 +57,7 @@ def run(self, **passedkwargs):
                     # Get our own copy of the kwargs object so we can change it
                     testkwargs = deepcopy(passedkwargs)
                     # Merges dicts
-                    testkwargs = { **kwargs, **testkwargs }  # noqa
+                    testkwargs = dict(kwargs, **testkwargs) # noqa
 
                     # Get the arguments that the test functions support
                     runfunc = getattr(testpackage, testname)

ioos_qc/qartod.py

@@ -4,7 +4,6 @@
 import warnings
 from collections import namedtuple
 from numbers import Real as N
-from typing import Sequence, Tuple, Union, Dict
 
 import numpy as np
 import pandas as pd
@@ -46,8 +45,7 @@ def mapdates(dates):
             return np.array(dates, dtype='datetime64[ns]')
 
 
-def qartod_compare(vectors : Sequence[Sequence[N]]
-                   ) -> np.ma.MaskedArray:
+def qartod_compare(vectors):
     """Aggregates an array of flags by precedence into a single array.
 
     Args:
@@ -80,11 +78,7 @@ def qartod_compare(vectors : Sequence[Sequence[N]]
     return result
 
 
-def location_test(lon : Sequence[N],
-                  lat : Sequence[N],
-                  bbox : Tuple[N, N, N, N] = (-180, -90, 180, 90),
-                  range_max : N = None
-                  ) -> np.ma.core.MaskedArray:
+def location_test(lon, lat, bbox = (-180, -90, 180, 90), range_max = None):
     """Checks that a location is within reasonable bounds.
 
     Checks that longitude and latitude are within reasonable bounds defaulting
@@ -157,10 +151,7 @@ def location_test(lon : Sequence[N],
     return flag_arr.reshape(original_shape)
 
 
-def gross_range_test(inp : Sequence[N],
-                     fail_span : Tuple[N, N],
-                     suspect_span : Tuple[N, N] = None
-                     ) -> np.ma.core.MaskedArray:
+def gross_range_test(inp, fail_span, suspect_span = None):
     """Checks that values are within reasonable range bounds.
 
     Given a 2-tuple of minimum/maximum values, flag data outside of the given
@@ -240,7 +231,7 @@ def __init__(self, members=None):
     def members(self):
         return self._members
 
-    def values(self, tind : pd.Timestamp, zind=None):
+    def values(self, tind, zind=None):
         """
         Args:
             tind: Value to test for inclusion between time bounds
@@ -268,12 +259,7 @@ def values(self, tind : pd.Timestamp, zind=None):
                     span = m.vspan
         return span
 
-    def add(self,
-            tspan : Tuple[N, N],
-            vspan : Tuple[N, N],
-            zspan : Tuple[N, N] = None,
-            period : str = None
-            ) -> None:
+    def add(self, tspan, vspan, zspan = None, period = None):
 
         assert isfixedlength(tspan, 2)
         # If period is defined, tspan is a numeric
@@ -376,11 +362,7 @@ def convert(config):
         return c
 
 
-def climatology_test(config : Union[ClimatologyConfig, Sequence[Dict[str, Tuple]]],
-                     inp : Sequence[N],
-                     tinp : Sequence[N],
-                     zinp : Sequence[N],
-                     ) -> np.ma.core.MaskedArray:
+def climatology_test(config, inp, tinp, zinp):
     """Checks that values are within reasonable range bounds and flags as SUSPECT.
 
     Data for which no ClimatologyConfig member exists is marked as UNKNOWN.
@@ -423,10 +405,7 @@ def climatology_test(config : Union[ClimatologyConfig, Sequence[Dict[str, Tuple]
     return flag_arr.reshape(original_shape)
 
 
-def spike_test(inp : Sequence[N],
-               suspect_threshold: N,
-               fail_threshold: N
-               ) -> np.ma.core.MaskedArray:
+def spike_test(inp, suspect_threshold, fail_threshold):
     """Check for spikes by checking neighboring data against thresholds
 
     Determine if there is a spike at data point n-1 by subtracting
@@ -479,10 +458,7 @@ def spike_test(inp : Sequence[N],
     return flag_arr.reshape(original_shape)
 
 
-def rate_of_change_test(inp : Sequence[N],
-                        tinp : Sequence[N],
-                        threshold : float
-                        ) -> np.ma.core.MaskedArray:
+def rate_of_change_test(inp, tinp, threshold):
     """Checks the first order difference of a series of values to see if
     there are any values exceeding a threshold defined by the inputs.
     These are then marked as SUSPECT.  It is up to the test operator
@@ -528,12 +504,7 @@ def rate_of_change_test(inp : Sequence[N],
     return flag_arr.reshape(original_shape)
 
 
-def flat_line_test(inp: Sequence[N],
-                   tinp: Sequence[N],
-                   suspect_threshold: int,
-                   fail_threshold: int,
-                   tolerance: N = 0
-                   ) -> np.ma.MaskedArray:
+def flat_line_test(inp, tinp, suspect_threshold, fail_threshold, tolerance = 0):
     """Check for consecutively repeated values within a tolerance.
     Missing and masked data is flagged as UNKNOWN.
     More information: https://github.com/ioos/ioos_qc/pull/11
@@ -614,10 +585,7 @@ def run_test(test_threshold, flag_value):
     return flag_arr.reshape(original_shape)
 
 
-def attenuated_signal_test(inp : Sequence[N],
-                           threshold : Tuple[N, N],
-                           check_type : str = 'std'
-                           ) -> np.ma.MaskedArray:
+def attenuated_signal_test(inp, threshold, check_type = 'std'):
     """Check for near-flat-line conditions using a range or standard deviation.
 
     Missing and masked data is flagged as UNKNOWN.

ioos_qc/utils.py

@@ -3,7 +3,6 @@
 import geojson
 import logging
 from datetime import datetime, date
-from typing import Union
 from numbers import Real
 
 import numpy as np
@@ -12,9 +11,7 @@
 L = logging.getLogger(__name__)  # noqa
 
 
-def isfixedlength(lst : Union[list, tuple],
-                  length : int
-                  ) -> bool:
+def isfixedlength(lst, length):
     if not isinstance(lst, (list, tuple)):
         raise ValueError('Required: list/tuple, Got: {}'.format(type(lst)))
 
@@ -38,8 +35,7 @@ def isnan(v):
     )
 
 
-def check_timestamps(times : np.ndarray,
-                     max_time_interval : N = None):
+def check_timestamps(times, max_time_interval = None):
     """Sanity checks for timestamp arrays
 
     Checks that the times supplied are in monotonically increasing
@@ -72,7 +68,7 @@ def check_timestamps(times : np.ndarray,
 
 def dict_update(d, u):
     # http://stackoverflow.com/a/3233356
-    from collections.abc import Mapping
+    from collections import Mapping
     for k, v in u.items():
         if isinstance(d, Mapping):
             if isinstance(v, Mapping):

requirements.txt

@@ -1,6 +1,8 @@
+funcsigs
 geojson
 netCDF4
 numpy>=1.14
+pathlib
 pygc
 ruamel.yaml
 simplejson

setup.py

@@ -14,7 +14,7 @@ def readme():
 reqs = [line.strip() for line in open('requirements.txt') if not line.startswith('#')]
 
 setup(
-    name                = 'ioos_qc',
+    name                = 'ioos_qc_py2',
     version             = version(),
     description         = 'IOOS QARTOD and Quality Control tests implemented in Python',
     long_description    = readme(),
@@ -29,8 +29,7 @@ def readme():
         'Intended Audience :: Developers',
         'Intended Audience :: Science/Research',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 2.7',
         'Topic :: Scientific/Engineering',
         'Topic :: Scientific/Engineering :: GIS',
         'Topic :: Scientific/Engineering :: Information Analysis',

tests/test_config.py

@@ -251,15 +251,15 @@ def test_climatology_json_conversion(self):
         json_climatology_config = ClimatologyConfig.convert(qc.config['qartod']['climatology_test']['config'])
         self._assert_cc_configs_equal(self.cc, json_climatology_config)
 
-    def _assert_cc_configs_equal(self, c1: ClimatologyConfig, c2: ClimatologyConfig):
+    def _assert_cc_configs_equal(self, c1, c2):
         assert len(c1.members) == len(c2.members)
         for idx in range(0, len(c1.members)):
             m1 = c1.members[idx]
             m2 = c2.members[idx]
-            assert m1.tspan == m2.tspan, f"{idx} tspan did not match"
-            assert m1.vspan == m2.vspan, f"{idx} vspan did not match"
-            assert m1.zspan == m2.zspan, f"{idx} zspan did not match"
-            assert m1.period == m2.period, f"{idx} period did not match"
+            assert m1.tspan == m2.tspan, "{} tspan did not match".format(idx)
+            assert m1.vspan == m2.vspan, "{} vspan did not match".format(idx)
+            assert m1.zspan == m2.zspan, "{} zspan did not match".format(idx)
+            assert m1.period == m2.period, "{} period did not match".format(idx)
 
 
 class TestReadNcConfig(unittest.TestCase):

tests/test_qartod.py

@@ -770,7 +770,7 @@ def setUp(self):
         self.times = np.arange('2015-01-01 00:00:00', '2015-01-01 06:00:00',
                                step=np.timedelta64(15, 'm'), dtype=np.datetime64)
         self.times_epoch_secs = [t.astype(int) for t in self.times]
-        self.threshold = 5 / 15 / 60  # 5 units per 15 minutes --> 5/15/60 units per second
+        self.threshold = 5.0 / 15 / 60  # 5 units per 15 minutes --> 5/15/60 units per second
 
     def test_rate_of_change(self):
         times = self.times