Detect and flatten nested WDL directories

src/cromshell/submit/command.py

@@ -4,6 +4,9 @@
 import logging
 from datetime import datetime
 from pathlib import Path, PurePath
+import tempfile
+
+import os
 
 import click
 import requests
@@ -65,6 +68,11 @@ def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation):
 
     http_utils.assert_can_communicate_with_server(config=config)
 
+    if io_utils.has_nested_dependencies(wdl):
+        tempdir = tempfile.TemporaryDirectory(prefix='cromshell_')
+        wdl = io_utils.flatten_nested_dependencies(tempdir, wdl)
+        dependencies_zip = tempdir.name
+
     if no_validation:
         LOGGER.info("Skipping WDL validation")
     else:

src/cromshell/utilities/io_utils.py

@@ -1,11 +1,13 @@
 import json
 import logging
 import re
+import os
 import shutil
 from contextlib import nullcontext
 from io import BytesIO
 from pathlib import Path
-from typing import BinaryIO, List, Union
+import tempfile
+from typing import BinaryIO, List, Union, Tuple
 from zipfile import ZIP_DEFLATED, ZipFile
 
 from pygments import formatters, highlight, lexers
@@ -101,6 +103,59 @@ def assert_path_is_not_empty(path: Union[str, Path], description: str) -> None:
             raise EOFError(f"ERROR: {description} is empty: {path}.")
 
 
+def has_nested_dependencies(wdl_path: str or Path) -> bool:
+    """Determine if a WDL has any nested imports."""
+
+    with open(wdl_path, 'r') as rf:
+        for l in rf:
+            if l.startswith('import'):
+                m = re.match(r'import "(.+)"', l)
+
+                if "../" in m.group(1):
+                    return True
+
+    return False
+
+
+def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path):
+    """Generate the filename to use for flattened WDL files."""
+
+    return os.path.join(
+        tempdir.name,
+        re.sub("^-", "", re.sub(os.path.sep, "-", os.path.dirname(wdl_path))) + '-' + os.path.basename(wdl_path)
+    )
+
+
+def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> str:
+    """Flatten a WDL directory structure and rewrite imports accordingly.
+
+    Return string representing the filesystem location of the rewritten WDL.
+    """
+
+    wdl_dir = os.path.dirname(wdl_path)
+
+    new_wdl_path = get_flattened_filename(tempdir, wdl_path)
+
+    with open(wdl_path, 'r') as rf, open(new_wdl_path, 'w') as wf:
+        for l in rf:
+            if l.startswith('import'):
+                m = re.match(r'import "(.+)"', l)
+
+                imported_wdl_path = os.path.abspath(os.path.join(wdl_dir, m.group(1)))
+                import_line = re.sub(m.group(1), os.path.basename(get_flattened_filename(tempdir, imported_wdl_path)), l)
+
+                if ' as ' in l:
+                    wf.write(import_line)
+                else:
+                    wf.write(f'{import_line.strip()} as {re.sub(".wdl", "", os.path.basename(imported_wdl_path))}\n')
+
+                flatten_nested_dependencies(tempdir, imported_wdl_path)
+            else:
+                wf.write(l)
+
+    return new_wdl_path
+
+
 def open_or_zip(path: Union[str, Path, None]) -> Union[nullcontext, BytesIO, BinaryIO]:
     """Return a context that may be used for reading the contents from the path.