diff --git a/README.md b/README.md index 05aa992db..c46c2aad5 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,7 @@ Unified Workflow Tools for use with UFS applications and beyond ## Documentation -Comprehensive documentation is available for [the development version](https://uwtools.readthedocs.io/en/main/) and for [the latest release](https://uwtools.readthedocs.io/en/stable/). +Comprehensive documentation is available for: + +* [The latest release](https://uwtools.readthedocs.io/en/stable/) +* [The development version](https://uwtools.readthedocs.io/en/main/) diff --git a/docs/conf.py b/docs/conf.py index db432053b..b19d1b6f5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,7 @@ html_static_path = ["static"] html_theme = "sphinx_rtd_theme" intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} +linkcheck_ignore = [r"https://github.com/.*#.*"] nitpick_ignore_regex = [("py:class", r"^uwtools\..*")] numfig = True numfig_format = {"figure": "Figure %s"} @@ -40,6 +41,7 @@ "coverage": ("https://coverage.readthedocs.io/en/7.3.4/%s", "%s"), "docformatter": ("https://docformatter.readthedocs.io/en/stable/%s", "%s"), "github-docs": ("https://docs.github.com/en/%s", "%s"), + "iotaa-readme": ("https://github.com/maddenp/iotaa/blob/main/README.md#%s", "%s"), "isort": ("https://pycqa.github.io/isort/%s", "%s"), "jinja2": ("https://jinja.palletsprojects.com/%s", "%s"), "jq": ("https://jqlang.github.io/jq/manual/v1.7/%s", "%s"), diff --git a/docs/sections/contributor_guide/developer_setup.rst b/docs/sections/contributor_guide/developer_setup.rst index 0e3b887ee..89bf2abb9 100644 --- a/docs/sections/contributor_guide/developer_setup.rst +++ b/docs/sections/contributor_guide/developer_setup.rst @@ -6,7 +6,7 @@ Creating a ``bash`` Development Shell If an existing conda (:miniforge:`Miniforge<>`, :miniconda:`Miniconda<>`, :anaconda:`Anaconda<>`, etc.) installation is available and writable, step 1 may be skipped. -.. include:: /shared/miniforge_instructions.rst +#. .. include:: /shared/miniforge_instructions.rst #. Install the :anaconda-condev:`condev package<>` into the ``base`` environment. diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 2d870b0fd..8e4336103 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -5,8 +5,8 @@ API cdeps chgres_cube config - esg_grid driver + esg_grid file filter_topo fv3 diff --git a/docs/sections/user_guide/cli/drivers/index.rst b/docs/sections/user_guide/cli/drivers/index.rst index 89ebd29ad..b61b8242b 100644 --- a/docs/sections/user_guide/cli/drivers/index.rst +++ b/docs/sections/user_guide/cli/drivers/index.rst @@ -1,3 +1,5 @@ +.. _drivers: + Drivers ======= diff --git a/docs/sections/user_guide/cli/tools/execute.rst b/docs/sections/user_guide/cli/tools/execute.rst new file mode 100644 index 000000000..d5d6ee945 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute.rst @@ -0,0 +1,64 @@ +``execute`` +=========== + +The ``uw`` mode for executing external drivers. + +.. literalinclude:: execute/help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: execute/help.out + :language: text + +For the three required arguments: + +* ``--module`` specifies the name of the module providing the driver. The name may be an absolute path (e.g. ``/path/to/driver.py``); a path relative to the current directory (e.g. ``driver.py``, ``../driver.py``, ``sub/dir/driver.py``); or a name appropriate to the Python ``import`` statement (e.g. ``driver``, ``my.package.driver``), provided the directory containing the module is on ``PYTHONPATH`` / ``sys.path``. +* ``--class`` specifies the name of a class in the above module that implements the driver, which should use one of the classes exported by ``uwtools.api.driver`` as its base class. +* ``--task`` specifies the name of a method in the above class that implements a :iotaa-readme:`task`, decorated with :iotaa-readme:`@task`, :iotaa-readme:`@tasks`, or :iotaa-readme:`@external`. + +.. _cli_execute_examples: + +Examples +^^^^^^^^ + +These examples use the following inputs: + +Module ``rand.py`` + +.. literalinclude:: execute/rand.py + :language: python + +Schema ``rand.jsonschema`` + +.. literalinclude:: execute/rand.jsonschema + :language: json + +Config ``rand.yaml`` + +.. literalinclude:: execute/rand.yaml + :language: yaml + +* Execute the external driver: + + .. literalinclude:: execute/execute.cmd + :language: text + :emphasize-lines: 2 + .. literalinclude:: execute/execute.out + :language: text + +* If the external driver does not accept an argument that was provided on the command line, it will exit with error. In this case, ``Rand`` inherits from parent class ``AssetsTimeInvariant``, which does not accept a ``cycle`` argument: + + .. literalinclude:: execute/bad-arg.cmd + :language: text + :emphasize-lines: 1 + .. literalinclude:: execute/bad-arg.out + :language: text + +* If the schema file for a driver resides in the same directory as its Python module and has the same filename prefix, as well as a ``.jsonschema`` suffix (e.g. ``rand.jsonschema`` alongside ``rand.py``) then the ``--schema-file`` argument is not required. However, ``--schema-file`` can be used to point to an alternate schema: + + .. literalinclude:: execute/alt-schema.cmd + :language: text + :emphasize-lines: 2 + .. literalinclude:: execute/alt-schema.out + :language: text + +* Other arguments behave identically to the same-named arguments to built-in ``uwtools`` drivers (see :ref:`drivers`). diff --git a/docs/sections/user_guide/cli/tools/execute/Makefile b/docs/sections/user_guide/cli/tools/execute/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/tools/execute/alt-schema.cmd b/docs/sections/user_guide/cli/tools/execute/alt-schema.cmd new file mode 100644 index 000000000..807ba8626 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/alt-schema.cmd @@ -0,0 +1,3 @@ +rm -rf /tmp/rand-alt +uw execute --module rand.py --classname Rand --task randfile --config-file alt.yaml --schema-file alt.schema +echo Random integer is $(cat /tmp/rand-alt/randint) diff --git a/docs/sections/user_guide/cli/tools/execute/alt-schema.out b/docs/sections/user_guide/cli/tools/execute/alt-schema.out new file mode 100644 index 000000000..9691a3987 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/alt-schema.out @@ -0,0 +1,7 @@ +[2024-08-08T23:46:07] INFO 0 UW schema-validation errors found +[2024-08-08T23:46:07] INFO rand Random-integer file: Initial state: Not Ready +[2024-08-08T23:46:07] INFO rand Random-integer file: Checking requirements +[2024-08-08T23:46:07] INFO rand Random-integer file: Requirement(s) ready +[2024-08-08T23:46:07] INFO rand Random-integer file: Executing +[2024-08-08T23:46:07] INFO rand Random-integer file: Final state: Ready +Random integer is 38 diff --git a/docs/sections/user_guide/cli/tools/execute/alt.schema b/docs/sections/user_guide/cli/tools/execute/alt.schema new file mode 120000 index 000000000..c2c2114d2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/alt.schema @@ -0,0 +1 @@ +rand.jsonschema \ No newline at end of file diff --git a/docs/sections/user_guide/cli/tools/execute/alt.yaml b/docs/sections/user_guide/cli/tools/execute/alt.yaml new file mode 100644 index 000000000..fb607348e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/alt.yaml @@ -0,0 +1,4 @@ +rand: + hi: 100 + lo: 1 + rundir: /tmp/rand-alt diff --git a/docs/sections/user_guide/cli/tools/execute/bad-arg.cmd b/docs/sections/user_guide/cli/tools/execute/bad-arg.cmd new file mode 100644 index 000000000..305495079 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/bad-arg.cmd @@ -0,0 +1 @@ +uw execute --module rand.py --classname Rand --task randfile --config-file rand.yaml --cycle 2024-08-08T12 diff --git a/docs/sections/user_guide/cli/tools/execute/bad-arg.out b/docs/sections/user_guide/cli/tools/execute/bad-arg.out new file mode 100644 index 000000000..87d692c74 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/bad-arg.out @@ -0,0 +1 @@ +[2024-08-08T23:46:07] ERROR Rand does not accept argument 'cycle' diff --git a/docs/sections/user_guide/cli/tools/execute/execute.cmd b/docs/sections/user_guide/cli/tools/execute/execute.cmd new file mode 100644 index 000000000..924853df1 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/execute.cmd @@ -0,0 +1,3 @@ +rm -rf /tmp/rand +uw execute --module rand.py --classname Rand --task randfile --config-file rand.yaml +echo Random integer is $(cat /tmp/rand/randint) diff --git a/docs/sections/user_guide/cli/tools/execute/execute.out b/docs/sections/user_guide/cli/tools/execute/execute.out new file mode 100644 index 000000000..3c291fc99 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/execute.out @@ -0,0 +1,7 @@ +[2024-08-08T23:46:07] INFO 0 UW schema-validation errors found +[2024-08-08T23:46:07] INFO rand Random-integer file: Initial state: Not Ready +[2024-08-08T23:46:07] INFO rand Random-integer file: Checking requirements +[2024-08-08T23:46:07] INFO rand Random-integer file: Requirement(s) ready +[2024-08-08T23:46:07] INFO rand Random-integer file: Executing +[2024-08-08T23:46:07] INFO rand Random-integer file: Final state: Ready +Random integer is 43 diff --git a/docs/sections/user_guide/cli/tools/execute/help.cmd b/docs/sections/user_guide/cli/tools/execute/help.cmd new file mode 100644 index 000000000..6bdaa1332 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/help.cmd @@ -0,0 +1 @@ +uw execute --help diff --git a/docs/sections/user_guide/cli/tools/execute/help.out b/docs/sections/user_guide/cli/tools/execute/help.out new file mode 100644 index 000000000..265292176 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/help.out @@ -0,0 +1,42 @@ +usage: uw execute --module MODULE --classname CLASSNAME --task TASK [-h] + [--version] [--config-file PATH] [--schema-file PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--batch] [--dry-run] + [--graph-file PATH] [--key-path KEY[.KEY...]] [--quiet] + [--verbose] + +Execute external driver. + +Required arguments: + --module MODULE + Path to driver module or name of module on sys.path + --classname CLASSNAME + Name of driver class + --task TASK + Driver task to execute + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --schema-file PATH + Path to schema file to use for validation + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-08-08T18) + --leadtime LEADTIME + The leadtime as hours[:minutes[:seconds]] + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --key-path KEY[.KEY...] + Dot-separated path of keys leading through the config to the driver's + configuration block + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages diff --git a/docs/sections/user_guide/cli/tools/execute/rand.jsonschema b/docs/sections/user_guide/cli/tools/execute/rand.jsonschema new file mode 100644 index 000000000..e48ce084d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/rand.jsonschema @@ -0,0 +1,28 @@ +{ + "properties": { + "rand": { + "additionalProperties": false, + "properties": { + "hi": { + "type": "integer" + }, + "lo": { + "type": "integer" + }, + "rundir": { + "type": "string" + } + }, + "required": [ + "lo", + "hi", + "rundir" + ], + "type": "object" + }, + "required": [ + "rand" + ], + "type": "object" + } +} diff --git a/docs/sections/user_guide/cli/tools/execute/rand.py b/docs/sections/user_guide/cli/tools/execute/rand.py new file mode 100644 index 000000000..2a8e10b57 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/rand.py @@ -0,0 +1,28 @@ +from random import randint + +from iotaa import asset, task + +from uwtools.api.driver import AssetsTimeInvariant +from uwtools.api.logging import use_uwtools_logger + +use_uwtools_logger() + + +class Rand(AssetsTimeInvariant): + + @task + def randfile(self): + """ + A file containing a random integer. + """ + path = self.rundir / "randint" + yield self.taskname("Random-integer file") + yield asset(path, path.is_file) + yield None + path.parent.mkdir(parents=True) + with open(path, "w", encoding="utf-8") as f: + print(randint(self.config["lo"], self.config["hi"]), file=f) + + @property + def driver_name(self): + return "rand" diff --git a/docs/sections/user_guide/cli/tools/execute/rand.yaml b/docs/sections/user_guide/cli/tools/execute/rand.yaml new file mode 100644 index 000000000..2fc7273ff --- /dev/null +++ b/docs/sections/user_guide/cli/tools/execute/rand.yaml @@ -0,0 +1,4 @@ +rand: + hi: 100 + lo: 1 + rundir: /tmp/rand diff --git a/docs/sections/user_guide/cli/tools/index.rst b/docs/sections/user_guide/cli/tools/index.rst index b2527e0ad..27d52c00f 100644 --- a/docs/sections/user_guide/cli/tools/index.rst +++ b/docs/sections/user_guide/cli/tools/index.rst @@ -5,6 +5,7 @@ Tools :maxdepth: 1 config + execute file rocoto template diff --git a/docs/sections/user_guide/installation.rst b/docs/sections/user_guide/installation.rst index 292ba138b..2cd4f61c3 100644 --- a/docs/sections/user_guide/installation.rst +++ b/docs/sections/user_guide/installation.rst @@ -38,7 +38,7 @@ To create a standalone conda environment providing ``uwtools``: Use a Fresh Miniforge Installation ---------------------------------- -.. include:: /shared/miniforge_instructions.rst +#. .. include:: /shared/miniforge_instructions.rst #. Continue with the `Use an Existing conda Installation`_ instructions. diff --git a/docs/shared/miniforge_instructions.rst b/docs/shared/miniforge_instructions.rst index 804086b3b..15daeccb7 100644 --- a/docs/shared/miniforge_instructions.rst +++ b/docs/shared/miniforge_instructions.rst @@ -1,4 +1,4 @@ -#. Visit the :miniforge:`Miniforge releases page` and obtain the URL and filename for the ``Miniforge3-[os]-[architecture].sh`` installer appropriate to your system, for example ``Miniforge3-Linux-x86_64.sh`` or ``Miniforge3-MacOSX-arm64.sh``. Download, install, and activate. Modify the ``$HOME/conda`` installation directory per your needs. +Visit the :miniforge:`Miniforge releases page` and obtain the URL and filename for the ``Miniforge3-[os]-[architecture].sh`` installer appropriate to your system, for example ``Miniforge3-Linux-x86_64.sh`` or ``Miniforge3-MacOSX-arm64.sh``. Download, install, and activate. Modify the ``$HOME/conda`` installation directory per your needs. .. code-block:: text diff --git a/src/uwtools/api/driver.py b/src/uwtools/api/driver.py index b4615c0cd..71f9e946f 100644 --- a/src/uwtools/api/driver.py +++ b/src/uwtools/api/driver.py @@ -61,8 +61,10 @@ def execute( :param stdin_ok: OK to read from stdin? :return: ``True`` if task completes without raising an exception. """ - if not (class_ := _get_driver_class(module, classname)): + class_, module_path = _get_driver_class(module, classname) + if not class_: return False + assert module_path is not None args = dict(locals()) accepted = set(getfullargspec(class_).args) non_optional = {STR.cycle, STR.leadtime} @@ -78,7 +80,7 @@ def execute( config=ensure_data_source(config, bool(stdin_ok)), dry_run=dry_run, key_path=key_path, - schema_file=schema_file or Path(module).with_suffix(".jsonschema"), + schema_file=schema_file or module_path.with_suffix(".jsonschema"), ) required = non_optional & accepted for arg in sorted([STR.batch, *required]): @@ -93,46 +95,50 @@ def execute( return True -def tasks(module: str, classname: str) -> dict[str, str]: +def tasks(module: Union[Path, str], classname: str) -> dict[str, str]: """ Returns a mapping from task names to their one-line descriptions. :param module: Name of driver module. :param classname: Name of driver class to instantiate. """ - if not (class_ := _get_driver_class(module, classname)): + class_, _ = _get_driver_class(module, classname) + if not class_: log.error("Could not get tasks from class %s in module %s", classname, module) return {} return _tasks(class_) -def _get_driver_class(module: Union[Path, str], classname: str) -> Optional[Type]: +def _get_driver_class( + module: Union[Path, str], classname: str +) -> tuple[Optional[Type], Optional[Path]]: """ Returns the driver class. :param module: Name of driver module to load. :param classname: Name of driver class to instantiate. """ - module = str(module) - if not (m := _get_driver_module_explicit(module)): - if not (m := _get_driver_module_implicit(module)): + if not (m := _get_driver_module_explicit(Path(module))): + if not (m := _get_driver_module_implicit(str(module))): log.error("Could not load module %s", module) - return None + return None, None + assert m.__file__ is not None + module_path = Path(m.__file__) if hasattr(m, classname): c: Type = getattr(m, classname) - return c + return c, module_path log.error("Module %s has no class %s", module, classname) - return None + return None, module_path -def _get_driver_module_explicit(module: str) -> Optional[ModuleType]: +def _get_driver_module_explicit(module: Path) -> Optional[ModuleType]: """ Returns the named module found via explicit lookup of given path. :param module: Name of driver module to load. """ log.debug("Loading module %s", module) - if spec := spec_from_file_location(Path(module).name, module): + if spec := spec_from_file_location(module.name, module): m = module_from_spec(spec) if loader := spec.loader: try: @@ -150,8 +156,8 @@ def _get_driver_module_implicit(module: str) -> Optional[ModuleType]: :param module: Name of driver module to load. """ + log.debug("Loading module %s from sys.path", module) try: - log.debug("Loading module %s from sys.path", module) return import_module(module) except Exception: # pylint: disable=broad-exception-caught return None diff --git a/src/uwtools/tests/api/test_driver.py b/src/uwtools/tests/api/test_driver.py index c22a1defe..1b49a9737 100644 --- a/src/uwtools/tests/api/test_driver.py +++ b/src/uwtools/tests/api/test_driver.py @@ -117,82 +117,86 @@ def test_tasks_fail_no_cycle(args, caplog, kwargs): assert logged(caplog, "%s requires argument '%s'" % (args.classname, "cycle")) -def test_tasks_pass(args): - tasks = driver_api.tasks(classname=args.classname, module=args.module) +@mark.parametrize("f", [Path, str]) +def test_tasks_pass(args, f): + tasks = driver_api.tasks(classname=args.classname, module=f(args.module)) assert tasks["eighty_eight"] == "88" def test__get_driver_class_explicit_fail_bad_class(caplog, args): log.setLevel(logging.DEBUG) bad_class = "BadClass" - c = driver_api._get_driver_class(classname=bad_class, module=args.module) + c, module_path = driver_api._get_driver_class(classname=bad_class, module=args.module) assert c is None + assert module_path == args.module assert logged(caplog, "Module %s has no class %s" % (args.module, bad_class)) def test__get_driver_class_explicit_fail_bad_name(caplog, args): log.setLevel(logging.DEBUG) - bad_name = "bad_name" - c = driver_api._get_driver_class(classname=args.classname, module=bad_name) + bad_name = Path("bad_name") + c, module_path = driver_api._get_driver_class(classname=args.classname, module=bad_name) assert c is None + assert module_path is None assert logged(caplog, "Could not load module %s" % bad_name) def test__get_driver_class_explicit_fail_bad_path(caplog, args, tmp_path): log.setLevel(logging.DEBUG) module = tmp_path / "not.py" - c = driver_api._get_driver_class(classname=args.classname, module=module) + c, module_path = driver_api._get_driver_class(classname=args.classname, module=module) assert c is None + assert module_path is None assert logged(caplog, "Could not load module %s" % module) def test__get_driver_class_explicit_fail_bad_spec(caplog, args): log.setLevel(logging.DEBUG) with patch.object(driver_api, "spec_from_file_location", return_value=None): - c = driver_api._get_driver_class(classname=args.classname, module=args.module) + c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) assert c is None + assert module_path is None assert logged(caplog, "Could not load module %s" % args.module) def test__get_driver_class_explicit_pass(args): log.setLevel(logging.DEBUG) - c = driver_api._get_driver_class(classname=args.classname, module=args.module) + c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) assert c assert c.__name__ == "TestDriver" + assert module_path == args.module def test__get_driver_class_implicit_pass(args): log.setLevel(logging.DEBUG) with patch.object(Path, "cwd", return_value=fixture_path()): - c = driver_api._get_driver_class(classname=args.classname, module=args.module) - assert c - assert c.__name__ == "TestDriver" + c, module_path = driver_api._get_driver_class(classname=args.classname, module=args.module) + assert c + assert c.__name__ == "TestDriver" + assert module_path == args.module def test__get_driver_module_explicit_absolute_fail(args): assert args.module.is_absolute() - module = str(args.module.with_suffix(".bad")) + module = args.module.with_suffix(".bad") assert not driver_api._get_driver_module_explicit(module=module) def test__get_driver_module_explicit_absolute_pass(args): assert args.module.is_absolute() - module = str(args.module) - assert driver_api._get_driver_module_explicit(module=module) + assert driver_api._get_driver_module_explicit(module=args.module) def test__get_driver_module_explicit_relative_fail(args): args.module = Path(os.path.relpath(args.module)).with_suffix(".bad") assert not args.module.is_absolute() - module = str(args.module) - assert not driver_api._get_driver_module_explicit(module=module) + assert not driver_api._get_driver_module_explicit(module=args.module) def test__get_driver_module_explicit_relative_pass(args): args.module = Path(os.path.relpath(args.module)) assert not args.module.is_absolute() - module = str(args.module) - assert driver_api._get_driver_module_explicit(module=module) + assert driver_api._get_driver_module_explicit(module=args.module) def test__get_driver_module_implicit_pass_full_package():