dev to master branch (#123)

* New click parsing auxiliary functions

* updated denoising cli

* fixed click --help

* updated datasets paths

* refactoring dexp click commands

* added cli tests

* refactored stabilization function

* fixed pytest skip cpu only

* updated dexp view

* deprecreated serve and remote datasets

* minor fixes

* updated tiff command

* removed serve from docs

* updated dexp registration

* adding dev branch

* adding dexp generic function command (#109)

* added dexp generic

* minor fixes

* Merging changes from master to dev (#110)

* Added sum of intensities to segmentation (#104)

* added sum of intensities to segmentation

* Fixing sum of intensities

* Added multi-scale dexp -> ome zarr conversion (#105)

* added multiscale ome zarr conversion

* added multiscale ome zarr conversion

* added channel scaling

* fixed tuple error

* fix denoising patch search and added aux function to ds writing

* added time translation

* Improved dexp segm and view (#108)

* improved dexp segm and view

* minor changes to segm post-filtering and view

* Adding background command (#112)

* added skimage/cucim import and refactored backend detection

* moved generic parsing functions to parse module

* added background removal command

* refactored scatter_gather args order

* fixing channels reduction

* dexp crop bug fix

* cleaning up dexp cli (#113)

* removed isonet command

* updated extract-psf command

* refactoring deskew

* updated zarr dataset to be multi-process safe

* updated projrender command

* minor fixes

* fixed tensorstore -> numpy loading

* [fix] tensorstore open relative path (#115)

* fixing tensorstore loading error

* black update

* fixing tensorstore version (rollback)

Co-authored-by: Jordao Bragantini <[email protected]>

* [feature] added clear control time metadata (#116)

* added cc time metadata

* numpy and numba compatibility change

* moving backend dispatch function (#118)

* refactoring deconv cli

* refactoring deconv cli (#121)

* [feature] refactoring fuse command (#122)

* refactoring deconv cli

* refactored fusion command

* updated docs

.github/workflows/pull_requests.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - dev
 
 jobs:
   test:

.pre-commit-config.yaml

@@ -41,7 +41,7 @@ repos:
                --per-file-ignores, '__init__.py:F401 dexp/cli/*.py:E501']
 
   - repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.3.0
     hooks:
       - id: black
 

dexp/cli/dexp_commands/_test/test_cli.py

@@ -0,0 +1,96 @@
+import subprocess
+from typing import Sequence
+
+import pytest
+
+from dexp.utils.testing import cupy_only
+
+
+@pytest.mark.parametrize(
+    "dexp_zarr_path",
+    [dict(dataset_type="nuclei", dtype="uint16")],
+    indirect=True,
+)
+@pytest.mark.parametrize(
+    "command",
+    [
+        ["info"],
+        ["check"],
+        ["background", "-c", "image", "-a", "binary_area_threshold=500"],
+        ["copy", "-wk", "2", "-s", "[:,:,40:240,40:240]"],
+        ["deconv", "-c", "image", "-m", "lr", "-bp", "wb", "-i", "3"],
+        ["deconv", "-c", "image", "-m", "admm", "-w"],
+        ["denoise", "-c", "image"],
+        ["extract-psf", "-pt", "100", "-c", "image"],
+        ["generic", "-f", "gaussian_filter", "-pkg", "cupyx.scipy.ndimage", "-ts", "100", "-a", "sigma=1,2,2"],
+        ["histogram", "-c", "image", "-m", "0"],
+        ["projrender", "-c", "image", "-lt", "test projrender", "-lsi", "0.5"],
+        ["segment", "-dc", "image"],
+        ["tiff", "--split", "-wk", "2"],
+        ["tiff", "-p", "0", "-wk", "2"],
+        ["view", "-q", "-d", "2"],
+    ],
+)
+@cupy_only
+def test_cli_commands_nuclei_dataset(command: Sequence[str], dexp_zarr_path: str) -> None:
+    assert subprocess.run(["dexp"] + command + [dexp_zarr_path]).returncode == 0
+
+
+@pytest.mark.parametrize(
+    "dexp_zarr_path",
+    [dict(dataset_type="nuclei", n_time_pts=10, dtype="uint16")],
+    indirect=True,
+)
+@pytest.mark.parametrize("command", [["stabilize", "-mr", "2", "-wk", "2"]])
+@cupy_only
+def test_cli_commands_long_nuclei_dataset(command: Sequence[str], dexp_zarr_path: str) -> None:
+    assert subprocess.run(["dexp"] + command + [dexp_zarr_path]).returncode == 0
+
+
+@pytest.mark.parametrize(
+    "dexp_zarr_path",
+    [dict(dataset_type="fusion", dtype="uint16")],
+    indirect=True,
+)
+@pytest.mark.parametrize(
+    "command",
+    [
+        ["register", "-c", "image-C0L0,image-C1L0"],
+        ["fuse", "-c", "image-C0L0,image-C1L0", "-lr"],  # must be after registration
+    ],
+)
+@cupy_only
+def test_cli_commands_fusion_dataset(command: Sequence[str], dexp_zarr_path: str) -> None:
+    assert subprocess.run(["dexp"] + command + [dexp_zarr_path]).returncode == 0
+
+
+@pytest.mark.parametrize(
+    "dexp_zarr_path",
+    [dict(dataset_type="nuclei-skewed", dtype="uint16")],
+    indirect=True,
+)
+@pytest.mark.parametrize(
+    "command",
+    [
+        ["deskew", "-c", "image", "-xx", "1", "-zz", "1", "-a", "45"],
+    ],
+)
+@cupy_only
+def test_cli_commands_skewed_dataset(command: Sequence[str], dexp_zarr_path: str) -> None:
+    assert subprocess.run(["dexp"] + command + [dexp_zarr_path]).returncode == 0
+
+
+@pytest.mark.parametrize(
+    "dexp_zarr_path",
+    [dict(dataset_type="fusion-skewed", dtype="uint16")],
+    indirect=True,
+)
+@pytest.mark.parametrize(
+    "command",
+    [
+        ["fuse", "-c", "image-C0L0,image-C1L0", "-m", "mvsols", "-zpa", "0, 0"],
+    ],
+)
+@cupy_only
+def test_cli_commands_fusion_skewed_dataset(command: Sequence[str], dexp_zarr_path: str) -> None:
+    assert subprocess.run(["dexp"] + command + [dexp_zarr_path]).returncode == 0

dexp/cli/dexp_commands/add.py

@@ -1,43 +1,51 @@
+from typing import Sequence
+
 import click
 from arbol.arbol import aprint, asection
 
 from dexp.cli.defaults import DEFAULT_STORE
-from dexp.cli.parsing import _get_output_path, _parse_channels
-from dexp.datasets.open_dataset import glob_datasets
+from dexp.cli.parsing import (
+    channels_option,
+    input_dataset_argument,
+    optional_channels_callback,
+)
+from dexp.datasets import ZDataset
 
 
 @click.command()
-@click.argument("input_paths", nargs=-1)  # ,  help='input path'
-@click.option("--output_path", "-o")  # , help='output path'
-@click.option("--channels", "-c", default=None, help="List of channels, all channels when ommited.")
+@input_dataset_argument()
+@channels_option()
+@click.option("--output-path", "-o", required=True)
 @click.option(
-    "--rename",
+    "--rename-channels",
     "-rc",
     default=None,
     help="You can rename channels: e.g. if channels are ‘channel1,anotherc’ then ‘gfp,rfp’ would rename the ‘channel1’ channel to ‘gfp’, and ‘anotherc’ to ‘rfp’ ",
+    callback=optional_channels_callback,
 )
 @click.option("--store", "-st", default=DEFAULT_STORE, help="Zarr store: ‘dir’, ‘ndir’, or ‘zip’", show_default=True)
 @click.option("--overwrite", "-w", is_flag=True, help="Forces overwrite of target", show_default=True)
 @click.option(
     "--projection", "-p/-np", is_flag=True, default=True, help="If flags should be copied.", show_default=True
 )
-def add(input_paths, output_path, channels, rename, store, overwrite, projection):
+def add(
+    input_dataset: ZDataset,
+    output_path: str,
+    channels: Sequence[str],
+    rename_channels: Sequence[str],
+    store: str,
+    overwrite: bool,
+    projection: bool,
+) -> None:
     """Adds the channels selected from INPUT_PATHS to the given output dataset (created if not existing)."""
 
-    input_dataset, input_paths = glob_datasets(input_paths)
-    output_path = _get_output_path(input_paths[0], output_path, "_add")
-    channels = _parse_channels(input_dataset, channels)
-
-    if rename is None:
-        rename = input_dataset.channels()
-    else:
-        rename = rename.split(",")
-
-    with asection(f"Adding channels: {channels} from: {input_paths} to {output_path}, with new names: {rename}"):
+    with asection(
+        f"Adding channels: {channels} from: {input_dataset.path} to {output_path}, with new names: {rename_channels}"
+    ):
         input_dataset.add_channels_to(
             output_path,
             channels=channels,
-            rename=rename,
+            rename=rename_channels,
             store=store,
             overwrite=overwrite,
             add_projections=projection,

dexp/cli/dexp_commands/background.py

@@ -0,0 +1,70 @@
+from typing import Optional, Sequence, Union
+
+import click
+from arbol import asection
+
+from dexp.cli.parsing import (
+    KwargsCommand,
+    args_option,
+    channels_option,
+    func_args_to_str,
+    input_dataset_argument,
+    multi_devices_option,
+    output_dataset_options,
+    parse_args_to_kwargs,
+    slicing_option,
+    validate_function_kwargs,
+)
+from dexp.datasets import BaseDataset, ZDataset
+from dexp.datasets.operations.background import dataset_background
+from dexp.processing.crop.background import foreground_mask
+
+
+@click.command(
+    cls=KwargsCommand,
+    context_settings=dict(ignore_unknown_options=True),
+    epilog=func_args_to_str(foreground_mask, ["array"]),
+)
+@input_dataset_argument()
+@output_dataset_options()
+@channels_option()
+@multi_devices_option()
+@slicing_option()
+@args_option()
+@click.option(
+    "--reference-channel", "-rc", type=str, default=None, help="Optional channel to use for background removal."
+)
+@click.option(
+    "--merge-channels",
+    "-mc",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Flag to indicate to merge (with addition) the image channels to detect the foreground.",
+)
+def background(
+    input_dataset: BaseDataset,
+    output_dataset: ZDataset,
+    channels: Sequence[str],
+    reference_channel: Optional[str],
+    merge_channels: bool,
+    devices: Union[str, Sequence[int]],
+    args: Sequence[str],
+) -> None:
+    """Remove background by detecting foreground regions using morphological analysis."""
+    kwargs = parse_args_to_kwargs(args)
+    validate_function_kwargs(foreground_mask, kwargs)
+
+    with asection(f"Removing background of {input_dataset.path}."):
+        dataset_background(
+            input_dataset=input_dataset,
+            output_dataset=output_dataset,
+            channels=channels,
+            reference_channel=reference_channel,
+            merge_channels=merge_channels,
+            devices=devices,
+            **kwargs,
+        )
+
+    input_dataset.close()
+    output_dataset.close()

dexp/cli/dexp_commands/check.py

@@ -1,20 +1,19 @@
+from typing import Sequence
+
 import click
 from arbol.arbol import aprint, asection
 
-from dexp.cli.parsing import _parse_channels
-from dexp.datasets.open_dataset import glob_datasets
+from dexp.cli.parsing import channels_option, input_dataset_argument
+from dexp.datasets.zarr_dataset import ZDataset
 
 
 @click.command()
-@click.argument("input_paths", nargs=-1)  # ,  help='input path'
-@click.option("--channels", "-c", default=None, help="List of channels, all channels when ommited.")
-def check(input_paths, channels):
+@input_dataset_argument()
+@channels_option()
+def check(input_dataset: ZDataset, channels: Sequence[int]) -> None:
     """Checks the integrity of a dataset."""
 
-    input_dataset, input_paths = glob_datasets(input_paths)
-    channels = _parse_channels(input_dataset, channels)
-
-    with asection(f"checking integrity of datasets {input_paths}, channels: {channels}"):
+    with asection(f"checking integrity of datasets {input_dataset.path}, channels: {channels}"):
         result = input_dataset.check_integrity(channels)
         input_dataset.close()