Merge pull request #143 from ECCO-GROUP/changes_for_processing

Changes for processing

ecco_v4_py/ecco_utils.py

@@ -58,16 +58,20 @@ def add_global_metadata(metadata, G, dataset_dim, less_output=True):
         # if we do add the field, we have to convert to the
         # appropriate data type
         if add_field == True:
-            if mtype == 's':
+            if isinstance(mc['value'], str) and 'TBD_' in mc['value']:
                 G.attrs[mname] = mc['value']
-            elif mtype == 'f':
-                G.attrs[mname] = float(mc['value'])
-            elif mtype == 'i':
-                G.attrs[mname] = np.int32(mc['value'])
             else:
-                print('INVALID MTYPE ! ', mtype)
+                if mtype == 's':
+                    G.attrs[mname] = mc['value']
+                elif mtype == 'f':
+                    G.attrs[mname] = float(mc['value'])
+                elif mtype == 'i':
+                    G.attrs[mname] = np.int32(mc['value'])
+                else:
+                    print('INVALID MTYPE ! ', mtype)
         else:
-            print('\t> not adding ', mc)
+            if not less_output:
+                print('\t> not adding ', mc)
 
     return G
 
@@ -95,7 +99,8 @@ def add_coordinate_metadata(metadata_dict, G, less_output=True):
                     if not less_output:
                         print('\t',m_key, ':', mv[m_key])
         else:
-            print('...... no metadata found in dictionary')
+            if not less_output:
+                print('...... no metadata found in dictionary')
 
     return G
 
@@ -115,7 +120,8 @@ def add_variable_metadata(variable_metadata_dict, G, \
         mv = find_metadata_in_json_dictionary(var, 'name', variable_metadata_dict)
 
         if len(mv) == 0:
-            print('...... no metadata found in dictionary')
+            if not less_output:
+                print('...... no metadata found in dictionary')
 
         else:
             # loop through each key, add if not on exclude list

ecco_v4_py/read_bin_llc.py

@@ -13,6 +13,7 @@
 
 from __future__ import division,print_function
 from xmitgcm import open_mdsdataset
+from xmitgcm.file_utils import clear_cache
 import xmitgcm
 import numpy as np
 import xarray as xr
@@ -51,6 +52,7 @@ def load_ecco_vars_from_mds(mds_var_dir,
                             mds_datatype = '>f4',
                             llc_method = 'bigchunks',
                             less_output=True,
+                            read_grid=True,
                             **kwargs):
 
     """
@@ -190,6 +192,8 @@ def load_ecco_vars_from_mds(mds_var_dir,
         str(start_day)  + ' ' +  str(start_hour) + ':' +  \
         str(start_min)  + ':' + str(start_sec)
 
+    # clear cache so listdir() will find the newly downloaded mds files for the current timestep
+    clear_cache()
 
     if model_time_steps_to_load == 'all':
         if not less_output:
@@ -200,7 +204,7 @@ def load_ecco_vars_from_mds(mds_var_dir,
 
         ecco_dataset = open_mdsdataset(data_dir = mds_var_dir,
                                        grid_dir = mds_grid_dir,
-                                       read_grid = True,
+                                       read_grid = read_grid,
                                        prefix = mds_files,
                                        geometry = 'llc',
                                        iters = 'all',
@@ -222,7 +226,7 @@ def load_ecco_vars_from_mds(mds_var_dir,
         if isinstance(model_time_steps_to_load, list):
             ecco_dataset = open_mdsdataset(data_dir = mds_var_dir,
                                            grid_dir = mds_grid_dir,
-                                           read_grid = True,
+                                           read_grid = read_grid,
                                            prefix = mds_files,
                                            geometry = 'llc',
                                            iters = model_time_steps_to_load,