From 314b181130c47a384b2975720a3a3a1cd808010c Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 00:34:19 -0400 Subject: [PATCH 1/7] Detect and flatten nested WDL directories to make execution compatible with Cromwell. --- src/cromshell/submit/command.py | 8 ++++ src/cromshell/utilities/io_utils.py | 57 ++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/cromshell/submit/command.py b/src/cromshell/submit/command.py index c75e9b7c..e6f9932b 100644 --- a/src/cromshell/submit/command.py +++ b/src/cromshell/submit/command.py @@ -4,6 +4,9 @@ import logging from datetime import datetime from pathlib import Path, PurePath +import tempfile + +import os import click import requests @@ -65,6 +68,11 @@ def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation): http_utils.assert_can_communicate_with_server(config=config) + if io_utils.has_nested_dependencies(wdl): + tempdir = tempfile.TemporaryDirectory(prefix='cromshell_') + wdl = io_utils.flatten_nested_dependencies(tempdir, wdl) + dependencies_zip = tempdir.name + if no_validation: LOGGER.info("Skipping WDL validation") else: diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index 022380aa..db99ea3c 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -1,11 +1,13 @@ import json import logging import re +import os import shutil from contextlib import nullcontext from io import BytesIO from pathlib import Path -from typing import BinaryIO, List, Union +import tempfile +from typing import BinaryIO, List, Union, Tuple from zipfile import ZIP_DEFLATED, ZipFile from pygments import formatters, highlight, lexers @@ -101,6 +103,59 @@ def assert_path_is_not_empty(path: Union[str, Path], description: str) -> None: raise EOFError(f"ERROR: {description} is empty: {path}.") +def has_nested_dependencies(wdl_path: str or Path) -> bool: + """Determine if a WDL has any nested imports.""" + + with open(wdl_path, 'r') as rf: + for l in rf: + if l.startswith('import'): + m = re.match(r'import "(.+)"', l) + + if "../" in m.group(1): + return True + + return False + + +def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path): + """Generate the filename to use for flattened WDL files.""" + + return os.path.join( + tempdir.name, + re.sub("^-", "", re.sub(os.path.sep, "-", os.path.dirname(wdl_path))) + '-' + os.path.basename(wdl_path) + ) + + +def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> str: + """Flatten a WDL directory structure and rewrite imports accordingly. + + Return string representing the filesystem location of the rewritten WDL. + """ + + wdl_dir = os.path.dirname(wdl_path) + + new_wdl_path = get_flattened_filename(tempdir, wdl_path) + + with open(wdl_path, 'r') as rf, open(new_wdl_path, 'w') as wf: + for l in rf: + if l.startswith('import'): + m = re.match(r'import "(.+)"', l) + + imported_wdl_path = os.path.abspath(os.path.join(wdl_dir, m.group(1))) + import_line = re.sub(m.group(1), os.path.basename(get_flattened_filename(tempdir, imported_wdl_path)), l) + + if ' as ' in l: + wf.write(import_line) + else: + wf.write(f'{import_line} as {re.sub(".wdl", "", os.path.basename(imported_wdl_path))}') + + flatten_nested_dependencies(tempdir, imported_wdl_path) + else: + wf.write(l) + + return new_wdl_path + + def open_or_zip(path: Union[str, Path, None]) -> Union[nullcontext, BytesIO, BinaryIO]: """Return a context that may be used for reading the contents from the path. From 9e5299124c58ba3b517c2d67b57e1b913e8dc6ff Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 00:56:41 -0400 Subject: [PATCH 2/7] Fixed a formatting mistake with newlines on rewritten imports --- src/cromshell/utilities/io_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index db99ea3c..f0782cee 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -147,7 +147,7 @@ def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: if ' as ' in l: wf.write(import_line) else: - wf.write(f'{import_line} as {re.sub(".wdl", "", os.path.basename(imported_wdl_path))}') + wf.write(f'{import_line.strip()} as {re.sub(".wdl", "", os.path.basename(imported_wdl_path))}\n') flatten_nested_dependencies(tempdir, imported_wdl_path) else: From fc02db4b9451c871c9b735a9b90bcdd2e5c146ff Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 22:45:11 -0400 Subject: [PATCH 3/7] Addressed PR comments from Beri --- src/cromshell/submit/command.py | 15 +++++++++++---- src/cromshell/utilities/io_utils.py | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/cromshell/submit/command.py b/src/cromshell/submit/command.py index e6f9932b..65ce9888 100644 --- a/src/cromshell/submit/command.py +++ b/src/cromshell/submit/command.py @@ -6,8 +6,6 @@ from pathlib import Path, PurePath import tempfile -import os - import click import requests from requests import Response @@ -60,16 +58,25 @@ class WorkflowStatusError(Exception): default=False, help="Do not check womtool for validation before submitting.", ) +@click.option( + "--do-not-flatten-wdls", + is_flag=True, + default=False, + help=".", +) @click.pass_obj -def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation): +def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation, do_not_flatten_wdls): """Submit a workflow and arguments to the Cromwell Server""" LOGGER.info("submit") http_utils.assert_can_communicate_with_server(config=config) - if io_utils.has_nested_dependencies(wdl): + if not do_not_flatten_wdls and io_utils.has_nested_dependencies(wdl): tempdir = tempfile.TemporaryDirectory(prefix='cromshell_') + + LOGGER.info(f"Flattening WDL structure to {tempdir.name}.") + wdl = io_utils.flatten_nested_dependencies(tempdir, wdl) dependencies_zip = tempdir.name diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index f0782cee..7d8c3c1c 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -1,7 +1,7 @@ import json import logging import re -import os +#import os import shutil from contextlib import nullcontext from io import BytesIO @@ -117,22 +117,22 @@ def has_nested_dependencies(wdl_path: str or Path) -> bool: return False -def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path): +def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> Path: """Generate the filename to use for flattened WDL files.""" - return os.path.join( - tempdir.name, - re.sub("^-", "", re.sub(os.path.sep, "-", os.path.dirname(wdl_path))) + '-' + os.path.basename(wdl_path) - ) + p = Path(wdl_path) + + return Path(tempdir.name + "/" + re.sub("^-", "", re.sub("/", "-", str(p.parent))) + '-' + str(p.name)) -def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> str: +def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> Path: """Flatten a WDL directory structure and rewrite imports accordingly. Return string representing the filesystem location of the rewritten WDL. """ - wdl_dir = os.path.dirname(wdl_path) + p = Path(wdl_path) + wdl_dir = p.parent new_wdl_path = get_flattened_filename(tempdir, wdl_path) @@ -141,13 +141,13 @@ def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: if l.startswith('import'): m = re.match(r'import "(.+)"', l) - imported_wdl_path = os.path.abspath(os.path.join(wdl_dir, m.group(1))) - import_line = re.sub(m.group(1), os.path.basename(get_flattened_filename(tempdir, imported_wdl_path)), l) + imported_wdl_path = (Path(wdl_dir) / m.group(1)).absolute() + import_line = re.sub(m.group(1), Path(get_flattened_filename(tempdir, imported_wdl_path)).name, l) if ' as ' in l: wf.write(import_line) else: - wf.write(f'{import_line.strip()} as {re.sub(".wdl", "", os.path.basename(imported_wdl_path))}\n') + wf.write(f'{import_line.strip()} as {re.sub(".wdl", "", Path(imported_wdl_path).name)}\n') flatten_nested_dependencies(tempdir, imported_wdl_path) else: From c4260c6634829971ca911a01019f53c1adcbf8d3 Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 22:46:19 -0400 Subject: [PATCH 4/7] Update src/cromshell/utilities/io_utils.py Co-authored-by: bshifaw --- src/cromshell/utilities/io_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index 7d8c3c1c..622de7e1 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -118,7 +118,11 @@ def has_nested_dependencies(wdl_path: str or Path) -> bool: def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> Path: - """Generate the filename to use for flattened WDL files.""" + """ Generate hyphen-separated path to use for flattened WDL file path. + For example: + tempdir: /path/2/tempdir/ and wdl_path: /dir/path/2/wdl.wdl + returns: /path/2/tempdir/dir-path-2-wdl.wdl + """ p = Path(wdl_path) From 61a6d7171c4957c45129c3b82e08e4fb6c2c77ca Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 22:46:41 -0400 Subject: [PATCH 5/7] Update src/cromshell/utilities/io_utils.py Co-authored-by: bshifaw --- src/cromshell/utilities/io_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index 622de7e1..1a227e7c 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -144,7 +144,7 @@ def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: for l in rf: if l.startswith('import'): m = re.match(r'import "(.+)"', l) - + imported_wdl_name = m.group(1) imported_wdl_path = (Path(wdl_dir) / m.group(1)).absolute() import_line = re.sub(m.group(1), Path(get_flattened_filename(tempdir, imported_wdl_path)).name, l) From 7c27e8019d6c015365f0f78e1e02203ed8b0494f Mon Sep 17 00:00:00 2001 From: Kiran V Garimella Date: Wed, 26 Jul 2023 22:58:46 -0400 Subject: [PATCH 6/7] Fixed formatting issues --- src/cromshell/submit/command.py | 14 +++++-- src/cromshell/utilities/io_utils.py | 64 ++++++++++++++++++----------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/cromshell/submit/command.py b/src/cromshell/submit/command.py index 65ce9888..3cee26ee 100644 --- a/src/cromshell/submit/command.py +++ b/src/cromshell/submit/command.py @@ -2,9 +2,9 @@ import csv import json import logging +import tempfile from datetime import datetime from pathlib import Path, PurePath -import tempfile import click import requests @@ -65,7 +65,15 @@ class WorkflowStatusError(Exception): help=".", ) @click.pass_obj -def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation, do_not_flatten_wdls): +def main( + config, + wdl, + wdl_json, + options_json, + dependencies_zip, + no_validation, + do_not_flatten_wdls, +): """Submit a workflow and arguments to the Cromwell Server""" LOGGER.info("submit") @@ -73,7 +81,7 @@ def main(config, wdl, wdl_json, options_json, dependencies_zip, no_validation, d http_utils.assert_can_communicate_with_server(config=config) if not do_not_flatten_wdls and io_utils.has_nested_dependencies(wdl): - tempdir = tempfile.TemporaryDirectory(prefix='cromshell_') + tempdir = tempfile.TemporaryDirectory(prefix="cromshell_") LOGGER.info(f"Flattening WDL structure to {tempdir.name}.") diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index 1a227e7c..958ff251 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -1,13 +1,14 @@ import json import logging import re -#import os + +# import os import shutil +import tempfile from contextlib import nullcontext from io import BytesIO from pathlib import Path -import tempfile -from typing import BinaryIO, List, Union, Tuple +from typing import BinaryIO, List, Union from zipfile import ZIP_DEFLATED, ZipFile from pygments import formatters, highlight, lexers @@ -106,30 +107,41 @@ def assert_path_is_not_empty(path: Union[str, Path], description: str) -> None: def has_nested_dependencies(wdl_path: str or Path) -> bool: """Determine if a WDL has any nested imports.""" - with open(wdl_path, 'r') as rf: - for l in rf: - if l.startswith('import'): - m = re.match(r'import "(.+)"', l) + with open(wdl_path, "r") as rf: + for line in rf: + if line.startswith("import"): + m = re.match(r'import "(.+)"', line) - if "../" in m.group(1): + imported_wdl_name = m.group(1) + if "../" in imported_wdl_name: return True return False -def get_flattened_filename(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> Path: - """ Generate hyphen-separated path to use for flattened WDL file path. - For example: +def get_flattened_filename( + tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path +) -> Path: + """Generate hyphen-separated path to use for flattened WDL file path. + For example: tempdir: /path/2/tempdir/ and wdl_path: /dir/path/2/wdl.wdl returns: /path/2/tempdir/dir-path-2-wdl.wdl """ p = Path(wdl_path) - return Path(tempdir.name + "/" + re.sub("^-", "", re.sub("/", "-", str(p.parent))) + '-' + str(p.name)) + return Path( + tempdir.name + + "/" + + re.sub("^-", "", re.sub("/", "-", str(p.parent))) + + "-" + + str(p.name) + ) -def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path) -> Path: +def flatten_nested_dependencies( + tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path +) -> Path: """Flatten a WDL directory structure and rewrite imports accordingly. Return string representing the filesystem location of the rewritten WDL. @@ -140,22 +152,28 @@ def flatten_nested_dependencies(tempdir: tempfile.TemporaryDirectory, wdl_path: new_wdl_path = get_flattened_filename(tempdir, wdl_path) - with open(wdl_path, 'r') as rf, open(new_wdl_path, 'w') as wf: - for l in rf: - if l.startswith('import'): - m = re.match(r'import "(.+)"', l) + with open(wdl_path, "r") as rf, open(new_wdl_path, "w") as wf: + for line in rf: + if line.startswith("import"): + m = re.match(r'import "(.+)"', line) imported_wdl_name = m.group(1) - imported_wdl_path = (Path(wdl_dir) / m.group(1)).absolute() - import_line = re.sub(m.group(1), Path(get_flattened_filename(tempdir, imported_wdl_path)).name, l) - - if ' as ' in l: + imported_wdl_path = (Path(wdl_dir) / imported_wdl_name).absolute() + import_line = re.sub( + imported_wdl_name, + Path(get_flattened_filename(tempdir, imported_wdl_path)).name, + line, + ) + + if " as " in line: wf.write(import_line) else: - wf.write(f'{import_line.strip()} as {re.sub(".wdl", "", Path(imported_wdl_path).name)}\n') + wf.write( + f'{import_line.strip()} as {re.sub(".wdl", "", Path(imported_wdl_path).name)}\n' + ) flatten_nested_dependencies(tempdir, imported_wdl_path) else: - wf.write(l) + wf.write(line) return new_wdl_path From d2244733372fbd898e65132392b09d8fbcc44646 Mon Sep 17 00:00:00 2001 From: bshifaw Date: Tue, 26 Sep 2023 20:02:38 -0400 Subject: [PATCH 7/7] Add tests for kvg flatten nested wdl dirs functions (#274) * minor updates to io_utils.py * added tests for flatten wdl functions * removed task from test workflow that imports its task --------- Co-authored-by: bshifaw --- src/cromshell/utilities/io_utils.py | 20 ++-- tests/conftest.py | 2 +- tests/unit/test_io_utils.py | 93 +++++++++++++++++++ .../helloWorld_with_imports.wdl | 42 +++++++++ .../wdl_with_imports/hello_world_task.wdl | 57 ++++++++++++ 5 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 tests/workflows/wdl_with_imports/helloWorld_with_imports.wdl create mode 100644 tests/workflows/wdl_with_imports/hello_world_task.wdl diff --git a/src/cromshell/utilities/io_utils.py b/src/cromshell/utilities/io_utils.py index 958ff251..6915b34c 100644 --- a/src/cromshell/utilities/io_utils.py +++ b/src/cromshell/utilities/io_utils.py @@ -1,8 +1,6 @@ import json import logging import re - -# import os import shutil import tempfile from contextlib import nullcontext @@ -119,9 +117,7 @@ def has_nested_dependencies(wdl_path: str or Path) -> bool: return False -def get_flattened_filename( - tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path -) -> Path: +def get_flattened_filename(tempdir: str, wdl_path: str or Path) -> Path: """Generate hyphen-separated path to use for flattened WDL file path. For example: tempdir: /path/2/tempdir/ and wdl_path: /dir/path/2/wdl.wdl @@ -131,7 +127,7 @@ def get_flattened_filename( p = Path(wdl_path) return Path( - tempdir.name + tempdir + "/" + re.sub("^-", "", re.sub("/", "-", str(p.parent))) + "-" @@ -140,27 +136,31 @@ def get_flattened_filename( def flatten_nested_dependencies( - tempdir: tempfile.TemporaryDirectory, wdl_path: str or Path + tempdir: tempfile.TemporaryDirectory, wdl_path: str ) -> Path: """Flatten a WDL directory structure and rewrite imports accordingly. Return string representing the filesystem location of the rewritten WDL. + + tempdir: /path/2/tempdir/ + wdl_path: /dir/path/2/wdl.wdl + returns: /path/2/tempdir/dir-path-2-wdl.wdl """ p = Path(wdl_path) wdl_dir = p.parent - new_wdl_path = get_flattened_filename(tempdir, wdl_path) + new_wdl_path = get_flattened_filename(tempdir.name, wdl_path) with open(wdl_path, "r") as rf, open(new_wdl_path, "w") as wf: for line in rf: if line.startswith("import"): m = re.match(r'import "(.+)"', line) imported_wdl_name = m.group(1) - imported_wdl_path = (Path(wdl_dir) / imported_wdl_name).absolute() + imported_wdl_path = (Path(wdl_dir) / imported_wdl_name).resolve() import_line = re.sub( imported_wdl_name, - Path(get_flattened_filename(tempdir, imported_wdl_path)).name, + Path(get_flattened_filename(tempdir.name, imported_wdl_path)).name, line, ) diff --git a/tests/conftest.py b/tests/conftest.py index e48ca85a..49f41d54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,7 @@ def local_cromshell_config_json(local_hidden_cromshell_folder): @pytest.fixture def test_workflows_path(): - return Path(__file__).joinpath("workflows/") + return Path(__file__).parent.joinpath("workflows/") @pytest.fixture diff --git a/tests/unit/test_io_utils.py b/tests/unit/test_io_utils.py index ce9c86b2..98ca53c2 100644 --- a/tests/unit/test_io_utils.py +++ b/tests/unit/test_io_utils.py @@ -1,9 +1,12 @@ import csv import io import os +import re import shutil +import tempfile from contextlib import redirect_stdout from pathlib import Path +from tempfile import NamedTemporaryFile from zipfile import ZipFile import pytest @@ -328,6 +331,96 @@ def test_update_all_workflow_database_tsv( ): assert row[column_to_update] == update_value + @pytest.mark.parametrize( + "wdl_content, expected_result", + [ + ('import "other.wdl"', False), # No nested import + ('import "../nested/other.wdl"', True), # Nested import + ('import "nested/other.wdl"', False), # Relative path, but not nested + ("task my_task { command { echo 'Hello, World!' } }", False), # No import + ( + 'import "../nested/other.wdl"\nimport "nested/another.wdl"', + True, + ), # Multiple imports, one nested + ], + ) + def test_has_nested_dependencies(self, wdl_content, expected_result): + # Create a temporary file with the provided WDL content + with NamedTemporaryFile(mode="w", delete=False) as temp_file: + temp_file.write(wdl_content) + + wdl_path = Path(temp_file.name) + + # Call the function with the temporary file path + result = io_utils.has_nested_dependencies(wdl_path) + + # Check if the result matches the expected outcome + assert result == expected_result + + # Clean up the temporary file + wdl_path.unlink() + + @pytest.mark.parametrize( + "wdl_path, flattened_wdl_file", + [ + ("/dir/path/2/wdl.wdl", "dir-path-2-wdl.wdl"), + ("/another/wdl.wdl", "another-wdl.wdl"), + ], + ) + def test_get_flattened_filename(self, wdl_path, flattened_wdl_file): + # Create a TemporaryDirectory to simulate tempdir + with tempfile.TemporaryDirectory() as tempdir: + # tempdir = Path(tempdir_name) + wdl_path = Path(wdl_path) + + # Call the function with the simulated tempdir and wdl_path + result = io_utils.get_flattened_filename(tempdir, wdl_path) + + # Check if the result matches the expected outcome + assert result == Path(tempdir).joinpath(flattened_wdl_file) + + # Define test cases using @pytest.mark.parametrize + @pytest.mark.parametrize( + "wdl_path, expected_file_content", + [ + ( + "wdl_with_imports/helloWorld_with_imports.wdl", + ["-helloWorld.wdl", "-wdl_with_imports-hello_world_task.wdl"], + ), + ], + ) + def test_flatten_nested_dependencies( + self, wdl_path, expected_file_content, test_workflows_path + ): + # Create a temporary directory to simulate tempdir + + tempdir = tempfile.TemporaryDirectory() + abs_wdl_path = test_workflows_path.joinpath(wdl_path) + + abs_wdl_path_str = str(abs_wdl_path.absolute()) + + # Call the function with the simulated tempdir and wdl_path + result_path = io_utils.flatten_nested_dependencies( + tempdir=tempdir, wdl_path=abs_wdl_path_str + ) + + # Check if the result matches the expected outcome + expected_result_path = Path(tempdir.name).joinpath( + re.sub("^-", "", re.sub("/", "-", str(abs_wdl_path))) + ) + assert result_path == expected_result_path + + # Check if the expected file content is in the result file + for expected_file_content_line in expected_file_content: + parsed_line = ( + re.sub("^-", "", re.sub("/", "-", str(abs_wdl_path.parents[1]))) + + expected_file_content_line + ) + assert parsed_line in result_path.read_text() + + # Clean up the temporary directory + tempdir.cleanup() + @pytest.fixture def mock_data_path(self): return Path(__file__).parent.joinpath("mock_data/") diff --git a/tests/workflows/wdl_with_imports/helloWorld_with_imports.wdl b/tests/workflows/wdl_with_imports/helloWorld_with_imports.wdl new file mode 100644 index 00000000..faf08a5e --- /dev/null +++ b/tests/workflows/wdl_with_imports/helloWorld_with_imports.wdl @@ -0,0 +1,42 @@ +import "../helloWorld.wdl" as HelloWorldWorkflow +import "hello_world_task.wdl" as helloWorldTask + +workflow HelloWorld { + meta { + workflow_description: "echos hello world" + } + parameter_meta { + # Description of inputs: + # Required: + docker: "Docker image in which to run" + # Optional: + mem: "Amount of memory to give to the machine running each task in this workflow." + preemptible_attempts: "Number of times to allow each task in this workflow to be preempted." + disk_space_gb: "Amount of storage disk space (in Gb) to give to each machine running each task in this workflow." + cpu: "Number of CPU cores to give to each machine running each task in this workflow." + boot_disk_size_gb: "Amount of boot disk space (in Gb) to give to each machine running each task in this workflow." + } + String docker + + Int? mem + Int? preemptible_attempts + Int? disk_space_gb + Int? cpu + Int? boot_disk_size_gb + + call helloWorldTask.HelloWorldTask { + input: + docker = docker, + mem = mem, + preemptible_attempts = preemptible_attempts, + disk_space_gb = disk_space_gb, + cpu = cpu, + boot_disk_size_gb = boot_disk_size_gb + } + + output { + File output_file = HelloWorldTask.output_file + } +} + + diff --git a/tests/workflows/wdl_with_imports/hello_world_task.wdl b/tests/workflows/wdl_with_imports/hello_world_task.wdl new file mode 100644 index 00000000..aafccb2f --- /dev/null +++ b/tests/workflows/wdl_with_imports/hello_world_task.wdl @@ -0,0 +1,57 @@ +task HelloWorldTask { + + # ------------------------------------------------ + # Input args: + + # Required: + + # Runtime Options: + String docker + Int? mem + Int? preemptible_attempts + Int? disk_space_gb + Int? cpu + Int? boot_disk_size_gb + + # ------------------------------------------------ + # Process input args: + + # ------------------------------------------------ + # Get machine settings: + Boolean use_ssd = false + + # You may have to change the following two parameter values depending on the task requirements + Int default_ram_mb = 3 * 1024 + # WARNING: In the workflow, you should calculate the disk space as an input to this task (disk_space_gb). Please see [TODO: Link from Jose] for examples. + Int default_disk_space_gb = 100 + + Int default_boot_disk_size_gb = 15 + + # Mem is in units of GB but our command and memory runtime values are in MB + Int machine_mem = if defined(mem) then mem * 1024 else default_ram_mb + Int command_mem = machine_mem - 1024 + + # ------------------------------------------------ + # Run our command: + command <<< + set -e + echo 'Hello World!' + >>> + + # ------------------------------------------------ + # Runtime settings: +# runtime { +# docker: docker +# memory: machine_mem + " MB" +# disks: "local-disk " + select_first([disk_space_gb, default_disk_space_gb]) + if use_ssd then " SSD" else " HDD" +# bootDiskSizeGb: select_first([boot_disk_size_gb, default_boot_disk_size_gb]) +# preemptible: 0 +# cpu: select_first([cpu, 1]) +# } + + # ------------------------------------------------ + # Outputs: + output { + File output_file = stdout() + } + }