Merge pull request #279 from GEOS-ESM/feature/jstassi/#278-GMAO_etc

Feature/jstassi/#278 gmao etc
GEOS-ESM · Sep 8, 2022 · caf468c · caf468c
2 parents 09629d8 + dcea4c7
commit caf468c
Show file tree

Hide file tree

Showing 19 changed files with 1,414 additions and 602 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add YAML validator
 
 ### Changed
+- Converted obsys_rc.py, check_obsysrc.py, and associated unittests to Python 3
+- Added features to cmpdir.pl (option to toggle between diff and xxdiff; added -B to -bwi flags)
 
 - Generated command line to run the program
 - changed the location of the temporary folder for remap_restarts MERRA-2 case

diff --git a/GMAO_etc/check_obsysrc.py b/GMAO_etc/check_obsysrc.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 "Check obsys_rc file against available data."
 
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
@@ -113,7 +113,7 @@ def check(filename="obsys.rc",
 
       # set thresh gap value
       #---------------------
-      if threshold.has_key("all"):
+      if "all" in threshold:
          threshHrs = threshold["all"]
       else:
          threshHrs = 1   # default to one hour
@@ -153,7 +153,7 @@ def check(filename="obsys.rc",
          start += "00"
          stop  += "00"
 
-         if dict(template_info_list).has_key((template, interval)):
+         if (template, interval) in dict(template_info_list):
             dict(template_info_list)[(template, interval)].append((start, stop))
          else:
             template_info_list.append(((template, interval), [(start, stop)]))
@@ -198,7 +198,7 @@ def check(filename="obsys.rc",
                raise Exception(msg.format(num_times, template))
             intervalMins = int(intervalMins)
 
-            hh = intervalMins / 60
+            hh = int(intervalMins / 60)
             if hh == 0:
                nn = intervalMins
             else:

diff --git a/GMAO_etc/cmpdir.pl b/GMAO_etc/cmpdir.pl
@@ -23,10 +23,11 @@
 
 # global variables
 #-----------------
-my ($bindiff, $bwiFLG, $debug, $cdoX, $delim, $diffFLGs, $dir1, $dir2);
+my ($bindiff, $BbwiFLG, $debug, $cdoX, $delim, $diffFLGs, $dir1, $dir2);
 my ($dirA, $dirB, $dirL1, $dirL2, $dmgetX, $filemode, $first, $follow);
 my ($ignoreFLG, $ignoreFile, $list, $listx, $quiet, $recurse, $sortFLG);
 my ($subdir, $tarfile_dirA, $tarfile_dirB, $tdirfile, $tmpdir, $verbose);
+my ($xxdiff_available, $xxFLG);
 my (%different, %diffsBIN, %diffsTXT, %dir_display, %filesize, %found);
 my (%identical, %ignore, %opts, %patterns, %vopts);
 my (@exclude, @extEXCL, @extINC, @fileIDs, @files, @files1, @files2);
@@ -56,7 +57,7 @@
 # purpose  - get input parameters and initialize global variables
 #=======================================================================
 sub init {
-    my ($binX, $etcflg, $fcstflg, $help, $rsflg, $runflg);
+    my ($binX, $etcflg, $fcstflg, $help, $rsflg, $runflg, $xxdiff);
     my ($arrAddr, $val, @values, @vlist);
 
     # get runtime flags
@@ -151,9 +152,22 @@ sub init {
     # initialize other variables
     #---------------------------
     $diffFLGs = "";
+    $xxFLG = 1;
     $first = 1;
     %ignore = ();
 
+    # check for xxdiff availability
+    #------------------------------
+    chomp($xxdiff = `which xxdiff`);
+    if (-x $xxdiff) {
+        $xxFLG = 1;
+        $xxdiff_available = 1;
+    }
+    else {
+        $xxFLG = 0;
+        $xxdiff_available = 0;
+    }
+
     # select program to use for $bindiff
     #-----------------------------------
     if (-x "/home/jstassi/bin/cdo") {
@@ -461,7 +475,7 @@ sub cmp_lists {
     $found{$lfile1} = $lfile2;
     $different{$lfile1} = $lfile2;
     $diffsTXT{"1"} = $lfile1;
-    $diffFLGs = "-bwi";
+    $diffFLGs = "-Bbwi";
     return;
 }
 
@@ -901,17 +915,20 @@ sub show_text_diffs {
             }
             print "\n";
         }
-        if ($diffFLGs) { printf $fmt1, "b", "turn OFF -bwi diff flag" }
-        else           { printf $fmt1, "b", "turn ON -bwi diff flag"  }
+        if ($diffFLGs) { printf $fmt1, "b", "turn OFF -Bbwi diff flag" }
+        else           { printf $fmt1, "b", "turn ON -Bbwi diff flag"  }
 
-        if ($ignoreFLG) { printf $fmt1, "e", "edit ignore diffs list" }
-
-        if ($ignoreFLG) { printf $fmt1, "i", "turn OFF ignore diffs"  }
+        if ($ignoreFLG) { printf $fmt1, "e", "edit ignore diffs list";
+                          printf $fmt1, "i", "turn OFF ignore diffs"  }
         else            { printf $fmt1, "i", "turn ON ignore diffs "  }
 
         if ($sortFLG) { printf $fmt1, "s", "turn OFF sorted diff" }
         else          { printf $fmt1, "s", "turn ON sorted diff"  }
 
+        if ($xxdiff_available) {
+            if ($xxFLG) { printf $fmt1, "x", "turn OFF xxdiff" }
+            else        { printf $fmt1, "x", "turn ON xxdiff" }
+        }
         print "\n";
         print "Make Selection: [$dflt] ";
         chomp($sel = <STDIN>);
@@ -942,12 +959,22 @@ sub show_text_diffs {
             next;
         }
 
-        # toggle diff -bwi flag
-        #----------------------
+        # toggle between xxdiff and diff comparisons
+        #-------------------------------------------
+        if ($sel eq "x") {
+            if ($xxdiff_available) {
+                $xxFLG = ! $xxFLG;
+            }
+            $num -= 1; next;
+
+        }
+
+        # toggle diff -Bbwi flag
+        #-----------------------
         if ($sel eq "b") {
-            $bwiFLG = ! $bwiFLG;
+            $BbwiFLG = ! $BbwiFLG;
             if ($diffFLGs) { $diffFLGs = ""     }
-            else           { $diffFLGs = "-bwi" }
+            else           { $diffFLGs = "-Bbwi" }
             $num -= 1; next;
         }
 
@@ -999,6 +1026,7 @@ sub display_text_diffs {
     my ($num, %diffs, $amperflag);
     my ($file1, $file2, $base1, $base2);
     my ($srt, $f1, $f2, $cnt, $diff1, $diff2, $var, $var_);
+    my ($status);
 
     $num = shift @_;
     %diffs = @_;
@@ -1050,14 +1078,19 @@ sub display_text_diffs {
             $var_ = "\\\$\{var" .$cnt ."\}";
 
             sed_subst($f1, $diff1, $var_);
-            system_("sed -i \"${cnt}i$var = $diff1\" $f1") && die "Error: sed cmd;";
-
             sed_subst($f2, $diff2, $var_);
-            system_("sed -i \"${cnt}i$var = $diff2\" $f2") && die "Error: sed cmd;";
+            if ($xxFLG) {
+                system_("sed -i \"${cnt}i$var = $diff1\" $f1")
+                    && die "Error: sed cmd;";
+                system_("sed -i \"${cnt}i$var = $diff2\" $f2")
+                    && die "Error: sed cmd;";
+            }
         }
         $cnt++;
-        system_("sed -i \"${cnt}i==================\" $f1") && die "Error: sed cmd;";
-        system_("sed -i \"${cnt}i==================\" $f2") && die "Error: sed cmd;";
+        system_("sed -i \"${cnt}i==================\" $f1")
+            && die "Error: sed cmd;";
+        system_("sed -i \"${cnt}i==================\" $f2")
+            && die "Error: sed cmd;";
         print "DONE\n";
     }
 
@@ -1069,7 +1102,12 @@ sub display_text_diffs {
     if ($diffs{"wait"}) { $amperflag = ""  }
     else                { $amperflag = "&" }
 
-    system_("xxdiff --text $diffFLGs $f1 $f2 $amperflag");
+    if ($xxFLG) { system_("xxdiff --text $diffFLGs $f1 $f2 $amperflag") }
+    else {
+        $status = system_("diff --text $diffFLGs $f1 $f2");
+        print "\nNO DIFFERENCES FOUND\n" unless $status;
+    }
+
 }
 
 #=======================================================================
@@ -1131,11 +1169,13 @@ sub diffs_to_ignore {
               # allow user to make menu choice from previous menu
               #--------------------------------------------------
               if    ($diffs12 eq "a") { next }
-              elsif ($diffs12 eq "")  { last }
               elsif ($diffs12 eq "0") { last outer }
               elsif ($diffs12 eq "c") { $sel = "c"; last }
               elsif ($diffs12 eq "r") { $sel = "r"; last }
-
+              elsif ($diffs12 eq "")  {
+                  if (%ignore) { last }
+                  else         { last outer }
+              }
               ($diff1, $diff2) = ();
               ($diff1, $diff2) = split /[$delim]/, $diffs12;
 

diff --git a/GMAO_etc/obsys_rc.py b/GMAO_etc/obsys_rc.py
@@ -105,7 +105,7 @@ def __read(self):
                         begin_found = True
 
                     if begin_found:
-                        if count.has_key(obsclass):
+                        if obsclass in count:
                             msg = "Duplicate obsclass found: {0}"
                             raise ValueError(msg.format(obsclass))
                         else:
@@ -149,7 +149,7 @@ def has(self, obsclass):
         """
         Return True if obsclass listed in self.__obsinfo; otherwise False.
         """
-        return dict(self.__obsinfo).has_key(obsclass)
+        return dict(self.__obsinfo).__contains__(obsclass)
 
     #.......................................................................
     def obsinfo(self):

diff --git a/GMAO_etc/unittests/outexp/obsys-test1.rc.err b/GMAO_etc/unittests/outexp/obsys-test1.rc.err
@@ -0,0 +1,163 @@
+#            NCCS Specific Version
+#
+# This observation system resource file database will track the whereabout of GAAS
+# observation files. It will be used by scripts require_obsys (hard check existence
+# of observation files) and acquire_obsys (will call acquire to get files).
+#
+# Syntax:
+# -------
+# BEGIN observation_file_class => StandardName
+#   startYYYYMMDD_HHz-endYYYYMMDD_HHz incrementHHMMSS remote_file_name_template
+# END
+#
+# Note: set "endYYYYMMDD_HHz" as "present" if new files added to system daily.
+# Note: replacing "present" (see note above) with future date
+#=======================================================================
+
+# AVHRR ascending and descending Synoptic npz files
+#--------------------------------------------------
+BEGIN patmosx_asc => patmosx_v05r02.asc.%y4%m2%d2_%h2z.npz
+  20020701_00z-20020825_21z 240000 archive/input/dao_ops/obs/reanalysis/patmosx/Level2/Synoptic/Y%y4/M%m2/D%d2/patmosx_v05r02.asc.%y4%m2%d2_%h2z.npz
+  #==========================
+  # INCORRECT INTERVAL
+  #--------------------------
+  # interval listed: 240000
+  # interval actual: 030000
+  #--------------------------
+
+  #==========================
+  # MORE DATA FOUND
+  #--------------------------
+  [20020826_00z - 20020831_21z]
+  #--------------------------
+
+END
+BEGIN patmosx_des => patmosx_v05r02.des.%y4%m2%d2_%h2z.npz
+  20020701_00z-20020825_21z 240000 archive/input/dao_ops/obs/reanalysis/patmosx/Level2/Synoptic/Y%y4/M%m2/D%d2/patmosx_v05r02.des.%y4%m2%d2_%h2z.npz
+  #==========================
+  # INCORRECT INTERVAL
+  #--------------------------
+  # interval listed: 240000
+  # interval actual: 030000
+  #--------------------------
+
+  #==========================
+  # MORE DATA FOUND
+  #--------------------------
+  [20020826_00z - 20020831_21z]
+  #--------------------------
+
+END
+
+# MISR over land
+#---------------
+BEGIN misr_F12_bright => misr_F12_0022.bright_tc8.obs.%y4%m2%d2.ods
+  20000301_00z-20000331_00z 240000 archive/input/dao_ops/obs/reanalysis/misr/Level2/ODS_Bright/Y%y4/M%m2/misr_F12_0022.bright_tc8.obs.%y4%m2%d2.ods
+  #==========================
+  # MISSING DATA
+  #--------------------------
+  [20000301_00z - 20000302_00z]
+  #--------------------------
+
+END
+
+# MODIS L2 Data, version 6
+#-------------------------
+BEGIN mod04_006_flk => MOD04_L2.A%y4%j3.%h2%n2.006.NRT.hdf
+  20130526_00z-20130601_21z 040000 archive/input/pzsst/obs/flk/modis.006/MOD04/%y4/%j3/MOD04_L2.A%y4%j3.%h2%n2.006.NRT.hdf
+  #==========================
+  # NO DATA FOUND!!!
+  #--------------------------
+
+  20170401_00z-20170531_21z 240000 archive/input/dao_ops/obs/flk/modis.006/MOD04/%y4/%j3/MOD04_L2.A%y4%j3.%h2%n2.006.NRT.hdf
+  #==========================
+  # INCORRECT INTERVAL
+  #--------------------------
+  # interval listed: 240000
+  # interval actual: 000500
+  #--------------------------
+
+  #==========================
+  # MISSING DATA
+  # data gaps > 60 mins
+  #--------------------------
+  [20170401_0320z - 20170401_0420z]
+  [20170401_0430z - 20170401_0540z]
+  [20170407_1415z - 20170407_1510z]
+  [20170408_0325z - 20170408_0420z]
+  [20170408_0445z - 20170408_0545z]
+  [20170412_0440z - 20170412_0545z]
+  [20170414_0110z - 20170414_0205z]
+  [20170416_0415z - 20170416_0535z]
+  [20170422_0305z - 20170422_0420z]
+  [20170423_0220z - 20170423_0325z]
+  [20170423_0510z - 20170423_0650z]
+  [20170424_0140z - 20170424_0240z]
+  [20170424_0315z - 20170424_2035z]
+  [20170426_0440z - 20170426_0535z]
+  [20170427_0320z - 20170427_0435z]
+  [20170428_0220z - 20170428_0340z]
+  [20170430_0110z - 20170430_0240z]
+  [20170504_2350z - 20170505_0045z]
+  [20170505_0305z - 20170505_0400z]
+  [20170507_0115z - 20170507_0235z]
+  [20170507_0430z - 20170507_0540z]
+  [20170507_0600z - 20170507_0700z]
+  [20170508_0615z - 20170508_0745z]
+  [20170509_0420z - 20170509_0550z]
+  [20170510_0130z - 20170510_0225z]
+  [20170510_0445z - 20170510_0545z]
+  [20170513_0215z - 20170513_0325z]
+  [20170513_0340z - 20170513_0435z]
+  [20170514_0120z - 20170514_0225z]
+  [20170516_0105z - 20170516_0205z]
+  [20170516_0245z - 20170516_0340z]
+  [20170520_0205z - 20170520_0305z]
+  [20170522_0525z - 20170522_0755z]
+  [20170523_0110z - 20170523_0225z]
+  [20170523_0240z - 20170523_0350z]
+  #--------------------------
+
+  #==========================
+  # MORE DATA FOUND
+  #--------------------------
+  [20170531_2105z - 20170531_2145z]
+  [20170531_2235z - 20170531_2325z]
+  #--------------------------
+
+  20180228_00z-20180326_21z 120000 archive/input/dmdyst/obs/flk/modis.006/MOD04/%y4/%j3/MOD04_L2.A%y4%j3.%h2%n2.006.NRT.hdf
+  #==========================
+  # NO DATA FOUND!!!
+  #--------------------------
+
+END
+
+BEGIN merra_satwnd18_pre-qc_bufr => satwnd.%y4%m2%d2.t18z.pre-qc.ublk
+  19770101_00z-19770131_18z 240000 archive/input/dao_ops/obs/merra/conv/prepbufr/satwnd/Y%y4/M%m2/MERG_SATWND_prepbufr_%y4%m2%d218
+  #==========================
+  # MISSING DATA
+  #--------------------------
+  [19770101_18z - 19770110_18z]
+  [19770131_18z]
+  #--------------------------
+
+  #==========================
+  # MORE DATA FOUND
+  #--------------------------
+  [19850901_18z - 19850902_18z]
+  [19850904_18z - 19850907_18z]
+  [19850909_18z - 19850917_18z]
+  [19850919_18z - 19850930_18z]
+  #--------------------------
+
+END
+
+# Nimbus-07 TOMS
+#---------------
+BEGIN nim07_tomseff_nc => tomseff.%y4%m2%d2.t%h2z.nc
+  19790101_00z-19930531_18z 060000 /archive/u/kwargan/data/TOMS/NETCDF/Y%y4/M%m2/TOMSeff.%y4%m2%d2_%h2z.nc
+  #==========================
+  # NO DATA FOUND!!!
+  #--------------------------
+
+END