UW-649 Docs for uw execute tool (#569)

README.md

@@ -6,4 +6,7 @@ Unified Workflow Tools for use with UFS applications and beyond
 
 ## Documentation
 
-Comprehensive documentation is available for [the development version](https://uwtools.readthedocs.io/en/main/) and for [the latest release](https://uwtools.readthedocs.io/en/stable/).
+Comprehensive documentation is available for:
+
+* [The latest release](https://uwtools.readthedocs.io/en/stable/)
+* [The development version](https://uwtools.readthedocs.io/en/main/)

docs/conf.py

@@ -20,6 +20,7 @@
 html_static_path = ["static"]
 html_theme = "sphinx_rtd_theme"
 intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
+linkcheck_ignore = [r"https://github.com/.*#.*"]
 nitpick_ignore_regex = [("py:class", r"^uwtools\..*")]
 numfig = True
 numfig_format = {"figure": "Figure %s"}
@@ -40,6 +41,7 @@
     "coverage": ("https://coverage.readthedocs.io/en/7.3.4/%s", "%s"),
     "docformatter": ("https://docformatter.readthedocs.io/en/stable/%s", "%s"),
     "github-docs": ("https://docs.github.com/en/%s", "%s"),
+    "iotaa-readme": ("https://github.com/maddenp/iotaa/blob/main/README.md#%s", "%s"),
     "isort": ("https://pycqa.github.io/isort/%s", "%s"),
     "jinja2": ("https://jinja.palletsprojects.com/%s", "%s"),
     "jq": ("https://jqlang.github.io/jq/manual/v1.7/%s", "%s"),

docs/sections/contributor_guide/developer_setup.rst

@@ -6,7 +6,7 @@ Creating a ``bash`` Development Shell
 
 If an existing conda (:miniforge:`Miniforge<>`, :miniconda:`Miniconda<>`, :anaconda:`Anaconda<>`, etc.) installation is available and writable, step 1 may be skipped.
 
-.. include:: /shared/miniforge_instructions.rst
+#. .. include:: /shared/miniforge_instructions.rst
 
 #. Install the :anaconda-condev:`condev package<>` into the ``base`` environment.
 

docs/sections/user_guide/api/index.rst

@@ -5,8 +5,8 @@ API
    cdeps
    chgres_cube
    config
-   esg_grid
    driver
+   esg_grid
    file
    filter_topo
    fv3

docs/sections/user_guide/cli/drivers/index.rst

@@ -1,3 +1,5 @@
+.. _drivers:
+
 Drivers
 =======
 

docs/sections/user_guide/cli/tools/execute.rst

@@ -0,0 +1,64 @@
+``execute``
+===========
+
+The ``uw`` mode for executing external drivers.
+
+.. literalinclude:: execute/help.cmd
+   :language: text
+   :emphasize-lines: 1
+.. literalinclude:: execute/help.out
+   :language: text
+
+For the three required arguments:
+
+* ``--module`` specifies the name of the module providing the driver. The name may be an absolute path (e.g. ``/path/to/driver.py``); a path relative to the current directory (e.g. ``driver.py``, ``../driver.py``, ``sub/dir/driver.py``); or a name appropriate to the Python ``import`` statement (e.g. ``driver``, ``my.package.driver``), provided the directory containing the module is on ``PYTHONPATH`` / ``sys.path``.
+* ``--class`` specifies the name of a class in the above module that implements the driver, which should use one of the classes exported by ``uwtools.api.driver`` as its base class.
+* ``--task`` specifies the name of a method in the above class that implements a :iotaa-readme:`task<tasks>`, decorated with :iotaa-readme:`@task<task>`, :iotaa-readme:`@tasks<tasks-1>`, or :iotaa-readme:`@external<external>`.
+
+.. _cli_execute_examples:
+
+Examples
+^^^^^^^^
+
+These examples use the following inputs:
+
+Module ``rand.py``
+
+.. literalinclude:: execute/rand.py
+   :language: python
+
+Schema ``rand.jsonschema``
+
+.. literalinclude:: execute/rand.jsonschema
+     :language: json
+
+Config ``rand.yaml``
+
+.. literalinclude:: execute/rand.yaml
+   :language: yaml
+
+* Execute the external driver:
+
+  .. literalinclude:: execute/execute.cmd
+     :language: text
+     :emphasize-lines: 2
+  .. literalinclude:: execute/execute.out
+     :language: text
+
+* If the external driver does not accept an argument that was provided on the command line, it will exit with error. In this case, ``Rand`` inherits from parent class ``AssetsTimeInvariant``, which does not accept a ``cycle`` argument:
+
+  .. literalinclude:: execute/bad-arg.cmd
+     :language: text
+     :emphasize-lines: 1
+  .. literalinclude:: execute/bad-arg.out
+     :language: text
+
+* If the schema file for a driver resides in the same directory as its Python module and has the same filename prefix, as well as a ``.jsonschema`` suffix (e.g. ``rand.jsonschema`` alongside ``rand.py``) then the ``--schema-file`` argument is not required. However, ``--schema-file`` can be used to point to an alternate schema:
+
+  .. literalinclude:: execute/alt-schema.cmd
+     :language: text
+     :emphasize-lines: 2
+  .. literalinclude:: execute/alt-schema.out
+     :language: text
+
+* Other arguments behave identically to the same-named arguments to built-in ``uwtools`` drivers (see :ref:`drivers`).

docs/sections/user_guide/cli/tools/execute/Makefile

@@ -0,0 +1 @@
+../../Makefile.outputs

docs/sections/user_guide/cli/tools/execute/alt-schema.cmd

@@ -0,0 +1,3 @@
+rm -rf /tmp/rand-alt
+uw execute --module rand.py --classname Rand --task randfile --config-file alt.yaml --schema-file alt.schema
+echo Random integer is $(cat /tmp/rand-alt/randint)

docs/sections/user_guide/cli/tools/execute/alt-schema.out

@@ -0,0 +1,7 @@
+[2024-08-08T23:46:07]     INFO 0 UW schema-validation errors found
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Initial state: Not Ready
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Checking requirements
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Requirement(s) ready
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Executing
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Final state: Ready
+Random integer is 38

docs/sections/user_guide/cli/tools/execute/alt.schema

@@ -0,0 +1 @@
+rand.jsonschema

docs/sections/user_guide/cli/tools/execute/alt.yaml

@@ -0,0 +1,4 @@
+rand:
+  hi: 100
+  lo: 1
+  rundir: /tmp/rand-alt

docs/sections/user_guide/cli/tools/execute/bad-arg.cmd

@@ -0,0 +1 @@
+uw execute --module rand.py --classname Rand --task randfile --config-file rand.yaml --cycle 2024-08-08T12

docs/sections/user_guide/cli/tools/execute/bad-arg.out

@@ -0,0 +1 @@
+[2024-08-08T23:46:07]    ERROR Rand does not accept argument 'cycle'

docs/sections/user_guide/cli/tools/execute/execute.cmd

@@ -0,0 +1,3 @@
+rm -rf /tmp/rand
+uw execute --module rand.py --classname Rand --task randfile --config-file rand.yaml
+echo Random integer is $(cat /tmp/rand/randint)

docs/sections/user_guide/cli/tools/execute/execute.out

@@ -0,0 +1,7 @@
+[2024-08-08T23:46:07]     INFO 0 UW schema-validation errors found
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Initial state: Not Ready
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Checking requirements
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Requirement(s) ready
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Executing
+[2024-08-08T23:46:07]     INFO rand Random-integer file: Final state: Ready
+Random integer is 43

docs/sections/user_guide/cli/tools/execute/help.cmd

@@ -0,0 +1 @@
+uw execute --help

docs/sections/user_guide/cli/tools/execute/help.out

@@ -0,0 +1,42 @@
+usage: uw execute --module MODULE --classname CLASSNAME --task TASK [-h]
+                  [--version] [--config-file PATH] [--schema-file PATH]
+                  [--cycle CYCLE] [--leadtime LEADTIME] [--batch] [--dry-run]
+                  [--graph-file PATH] [--key-path KEY[.KEY...]] [--quiet]
+                  [--verbose]
+
+Execute external driver.
+
+Required arguments:
+  --module MODULE
+      Path to driver module or name of module on sys.path
+  --classname CLASSNAME
+      Name of driver class
+  --task TASK
+      Driver task to execute
+
+Optional arguments:
+  -h, --help
+      Show help and exit
+  --version
+      Show version info and exit
+  --config-file PATH, -c PATH
+      Path to UW YAML config file (default: read from stdin)
+  --schema-file PATH
+      Path to schema file to use for validation
+  --cycle CYCLE
+      The cycle in ISO8601 format (e.g. 2024-08-08T18)
+  --leadtime LEADTIME
+      The leadtime as hours[:minutes[:seconds]]
+  --batch
+      Submit run to batch scheduler
+  --dry-run
+      Only log info, making no changes
+  --graph-file PATH
+      Path to Graphviz DOT output [experimental]
+  --key-path KEY[.KEY...]
+      Dot-separated path of keys leading through the config to the driver's
+      configuration block
+  --quiet, -q
+      Print no logging messages
+  --verbose, -v
+      Print all logging messages

docs/sections/user_guide/cli/tools/execute/rand.jsonschema

@@ -0,0 +1,28 @@
+{
+  "properties": {
+    "rand": {
+      "additionalProperties": false,
+      "properties": {
+        "hi": {
+          "type": "integer"
+        },
+        "lo": {
+          "type": "integer"
+        },
+        "rundir": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "lo",
+        "hi",
+        "rundir"
+      ],
+      "type": "object"
+    },
+    "required": [
+      "rand"
+    ],
+    "type": "object"
+  }
+}

docs/sections/user_guide/cli/tools/execute/rand.py

@@ -0,0 +1,28 @@
+from random import randint
+
+from iotaa import asset, task
+
+from uwtools.api.driver import AssetsTimeInvariant
+from uwtools.api.logging import use_uwtools_logger
+
+use_uwtools_logger()
+
+
+class Rand(AssetsTimeInvariant):
+
+    @task
+    def randfile(self):
+        """
+        A file containing a random integer.
+        """
+        path = self.rundir / "randint"
+        yield self.taskname("Random-integer file")
+        yield asset(path, path.is_file)
+        yield None
+        path.parent.mkdir(parents=True)
+        with open(path, "w", encoding="utf-8") as f:
+            print(randint(self.config["lo"], self.config["hi"]), file=f)
+
+    @property
+    def driver_name(self):
+        return "rand"

docs/sections/user_guide/cli/tools/execute/rand.yaml

@@ -0,0 +1,4 @@
+rand:
+  hi: 100
+  lo: 1
+  rundir: /tmp/rand

docs/sections/user_guide/cli/tools/index.rst

@@ -5,6 +5,7 @@ Tools
    :maxdepth: 1
 
    config
+   execute
    file
    rocoto
    template

docs/sections/user_guide/installation.rst

@@ -38,7 +38,7 @@ To create a standalone conda environment providing ``uwtools``:
 Use a Fresh Miniforge Installation
 ----------------------------------
 
-.. include:: /shared/miniforge_instructions.rst
+#. .. include:: /shared/miniforge_instructions.rst
 
 #. Continue with the `Use an Existing conda Installation`_ instructions.
 

docs/shared/miniforge_instructions.rst

@@ -1,4 +1,4 @@
-#. Visit the :miniforge:`Miniforge releases page<releases/latest>` and obtain the URL and filename for the ``Miniforge3-[os]-[architecture].sh`` installer appropriate to your system, for example ``Miniforge3-Linux-x86_64.sh`` or ``Miniforge3-MacOSX-arm64.sh``. Download, install, and activate. Modify the ``$HOME/conda`` installation directory per your needs.
+Visit the :miniforge:`Miniforge releases page<releases/latest>` and obtain the URL and filename for the ``Miniforge3-[os]-[architecture].sh`` installer appropriate to your system, for example ``Miniforge3-Linux-x86_64.sh`` or ``Miniforge3-MacOSX-arm64.sh``. Download, install, and activate. Modify the ``$HOME/conda`` installation directory per your needs.
 
    .. code-block:: text
 

src/uwtools/api/driver.py

@@ -61,8 +61,10 @@ def execute(
     :param stdin_ok: OK to read from stdin?
     :return: ``True`` if task completes without raising an exception.
     """
-    if not (class_ := _get_driver_class(module, classname)):
+    class_, module_path = _get_driver_class(module, classname)
+    if not class_:
         return False
+    assert module_path is not None
     args = dict(locals())
     accepted = set(getfullargspec(class_).args)
     non_optional = {STR.cycle, STR.leadtime}
@@ -78,7 +80,7 @@ def execute(
         config=ensure_data_source(config, bool(stdin_ok)),
         dry_run=dry_run,
         key_path=key_path,
-        schema_file=schema_file or Path(module).with_suffix(".jsonschema"),
+        schema_file=schema_file or module_path.with_suffix(".jsonschema"),
     )
     required = non_optional & accepted
     for arg in sorted([STR.batch, *required]):
@@ -93,46 +95,50 @@ def execute(
     return True
 
 
-def tasks(module: str, classname: str) -> dict[str, str]:
+def tasks(module: Union[Path, str], classname: str) -> dict[str, str]:
     """
     Returns a mapping from task names to their one-line descriptions.
 
     :param module: Name of driver module.
     :param classname: Name of driver class to instantiate.
     """
-    if not (class_ := _get_driver_class(module, classname)):
+    class_, _ = _get_driver_class(module, classname)
+    if not class_:
         log.error("Could not get tasks from class %s in module %s", classname, module)
         return {}
     return _tasks(class_)
 
 
-def _get_driver_class(module: Union[Path, str], classname: str) -> Optional[Type]:
+def _get_driver_class(
+    module: Union[Path, str], classname: str
+) -> tuple[Optional[Type], Optional[Path]]:
     """
     Returns the driver class.
 
     :param module: Name of driver module to load.
     :param classname: Name of driver class to instantiate.
     """
-    module = str(module)
-    if not (m := _get_driver_module_explicit(module)):
-        if not (m := _get_driver_module_implicit(module)):
+    if not (m := _get_driver_module_explicit(Path(module))):
+        if not (m := _get_driver_module_implicit(str(module))):
             log.error("Could not load module %s", module)
-            return None
+            return None, None
+    assert m.__file__ is not None
+    module_path = Path(m.__file__)
     if hasattr(m, classname):
         c: Type = getattr(m, classname)
-        return c
+        return c, module_path
     log.error("Module %s has no class %s", module, classname)
-    return None
+    return None, module_path
 
 
-def _get_driver_module_explicit(module: str) -> Optional[ModuleType]:
+def _get_driver_module_explicit(module: Path) -> Optional[ModuleType]:
     """
     Returns the named module found via explicit lookup of given path.
 
     :param module: Name of driver module to load.
     """
     log.debug("Loading module %s", module)
-    if spec := spec_from_file_location(Path(module).name, module):
+    if spec := spec_from_file_location(module.name, module):
         m = module_from_spec(spec)
         if loader := spec.loader:
             try:
@@ -150,8 +156,8 @@ def _get_driver_module_implicit(module: str) -> Optional[ModuleType]:
 
     :param module: Name of driver module to load.
     """
+    log.debug("Loading module %s from sys.path", module)
     try:
-        log.debug("Loading module %s from sys.path", module)
         return import_module(module)
     except Exception:  # pylint: disable=broad-exception-caught
         return None