fix: segregate ERA5Land scripting due to slowness

refs #321

scripts/RUN_0Z.sh

@@ -41,11 +41,7 @@ python process_nldasv2_noah.py $(date -u --date '5 days ago' +'%Y %m %d') &
 cd ../qc
 python check_n0q.py
 
-cd ../era5
-python fetch_era5.py $(date -u --date '6 days ago' +'%Y %m %d')
-
 cd ../climodat
-python era5land_extract.py --valid=$(date -u --date '7 days ago' +'%Y-%m-%d')
 python nldas_extract.py --valid=$(date -u --date '6 days ago' +'%Y-%m-%d')
 
 cd ../iemre

scripts/RUN_0Z_ERA5LAND.sh

@@ -0,0 +1,14 @@
+# ERA5Land is very slow to download, so we need to isolate this processing
+HH=$(date -u +%H)
+if [ "$HH" -ne "00" ]
+    then
+        exit
+fi
+D6=$(date -u --date '6 days ago' +'%Y-%m-%d')
+D7=$(date -u --date '7 days ago' +'%Y-%m-%d')
+
+cd era5
+python fetch_era5.py --date=$D6
+
+cd ../climodat
+python era5land_extract.py --date=$D7

scripts/climodat/era5land_extract.py

@@ -170,7 +170,7 @@ def do(dt):
 
 
 @click.command()
-@click.option("--valid", type=click.DateTime())
+@click.option("--date", "valid", type=click.DateTime())
 def main(valid):
     """Go Main Go"""
     do(valid.date())

scripts/crontab

@@ -8,6 +8,7 @@ S=/opt/iem/scripts
 
 #
 0 18,19 * * * cd $S; sh RUN_0Z.sh
+1 18,19 * * * cd $S; sh RUN_0Z_ERA5LAND.sh
 0 0 * * * cd $S; sh RUN_MIDNIGHT.sh
 30 4 * * * cd $S; sh RUN_CLIMODAT_STATE.sh
 0 12 * * * cd $S; sh RUN_NOON.sh

scripts/era5/fetch_era5.py

@@ -3,10 +3,10 @@
 Run from RUN_0Z.sh for 5 days ago.
 """
 import os
-import sys
 from datetime import timedelta
 
 import cdsapi
+import click
 import numpy as np
 from pyiem import iemre
 from pyiem.util import logger, ncopen, utc
@@ -141,18 +141,14 @@ def run(valid):
     os.unlink(ncfn)
 
 
-def main(argv):
+@click.command()
+@click.option("--date", "valid", required=True, type=click.DateTime())
+def main(valid):
     """Go!"""
-    valid = utc(*[int(a) for a in argv[1:]])
-    offsets = []
-    if len(argv) == 5:
-        offsets = [0]
-    elif len(argv) == 4:
-        # Best to run for 1z through 0z as 0z has the 24hr sum to consider
-        offsets = range(1, 25)
-    for offset in offsets:
+    valid = utc(valid.year, valid.month, valid.day)
+    for offset in range(1, 25):
         run(valid + timedelta(hours=offset))
 
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main()