From 61cf2dde936a0f7a27beab2db8d994e04474300f Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:29:11 -0700 Subject: [PATCH 01/13] default 4-state pol settings --- recOrder/tests/cli_tests/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/tests/cli_tests/test_settings.py b/recOrder/tests/cli_tests/test_settings.py index 8276d3eb..4e37bc19 100644 --- a/recOrder/tests/cli_tests/test_settings.py +++ b/recOrder/tests/cli_tests/test_settings.py @@ -9,7 +9,7 @@ def test_reconstruction_settings(): s = settings.ReconstructionSettings( birefringence=settings.BirefringenceSettings() ) - assert len(s.input_channel_names) == 5 + assert len(s.input_channel_names) == 4 assert s.birefringence.apply_inverse.background_path == "" assert s.phase == None assert s.fluorescence == None From b85ebbacdb4cf941c5de1ac6fa069a95594ade55 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:35:49 -0700 Subject: [PATCH 02/13] fix input channels in tests --- recOrder/tests/cli_tests/test_compute_tf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/recOrder/tests/cli_tests/test_compute_tf.py b/recOrder/tests/cli_tests/test_compute_tf.py index 0a6de72f..2d6aad0e 100644 --- a/recOrder/tests/cli_tests/test_compute_tf.py +++ b/recOrder/tests/cli_tests/test_compute_tf.py @@ -11,11 +11,11 @@ def test_compute_transfer(tmp_path, input_zarr): recon_settings = settings.ReconstructionSettings( - input_channel_names=[f"State{i}" for i in range(5)], - reconstruction_dimension=3, - birefringence=settings.BirefringenceSettings(), - phase=settings.PhaseSettings(), - ) + input_channel_names=[f"State{i}" for i in range(4)], + reconstruction_dimension=3, + birefringence=settings.BirefringenceSettings(), + phase=settings.PhaseSettings(), + ) config_path = tmp_path / "test.yml" utils.model_to_yaml(recon_settings, config_path) From 1e5639c846702d3e70642f2f06aeef7752e7e735 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:35:58 -0700 Subject: [PATCH 03/13] black format tests --- recOrder/tests/cli_tests/test_compute_tf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/recOrder/tests/cli_tests/test_compute_tf.py b/recOrder/tests/cli_tests/test_compute_tf.py index 2d6aad0e..2ad01835 100644 --- a/recOrder/tests/cli_tests/test_compute_tf.py +++ b/recOrder/tests/cli_tests/test_compute_tf.py @@ -21,7 +21,9 @@ def test_compute_transfer(tmp_path, input_zarr): path, _ = input_zarr runner = CliRunner() - result = runner.invoke(cli, ["compute-tf", str(path), "-c", str(config_path)]) + result = runner.invoke( + cli, ["compute-tf", str(path), "-c", str(config_path)] + ) assert result.exit_code == 0 @@ -45,10 +47,10 @@ def test_compute_transfer_blank_output(): def test_compute_transfer_output_file(tmp_path, input_zarr): recon_settings = settings.ReconstructionSettings( - input_channel_names=["TEST"], - reconstruction_dimension=3, - phase=settings.PhaseSettings(), - ) + input_channel_names=["TEST"], + reconstruction_dimension=3, + phase=settings.PhaseSettings(), + ) config_path = tmp_path / "test.yml" utils.model_to_yaml(recon_settings, config_path) From 74160c12de8198404d16e4a1ef065601d4fcf6e5 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:46:35 -0700 Subject: [PATCH 04/13] 5 channel pol + BF test input --- recOrder/tests/cli_tests/test_compute_tf.py | 2 +- recOrder/tests/conftest.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/recOrder/tests/cli_tests/test_compute_tf.py b/recOrder/tests/cli_tests/test_compute_tf.py index 2ad01835..d4d56328 100644 --- a/recOrder/tests/cli_tests/test_compute_tf.py +++ b/recOrder/tests/cli_tests/test_compute_tf.py @@ -47,7 +47,7 @@ def test_compute_transfer_blank_output(): def test_compute_transfer_output_file(tmp_path, input_zarr): recon_settings = settings.ReconstructionSettings( - input_channel_names=["TEST"], + input_channel_names=["BF"], reconstruction_dimension=3, phase=settings.PhaseSettings(), ) diff --git a/recOrder/tests/conftest.py b/recOrder/tests/conftest.py index e34d4127..28148d0c 100644 --- a/recOrder/tests/conftest.py +++ b/recOrder/tests/conftest.py @@ -12,9 +12,9 @@ def input_zarr(tmp_path): path, layout="fov", mode="w", - channel_names=["None"], + channel_names=[f"State{i}" for i in range(4)] + ["BF"], ) - dataset.create_zeros("0", (2, 1, 4, 5, 6), dtype=np.float32) + dataset.create_zeros("0", (2, 5, 4, 5, 6), dtype=np.float32) yield path, dataset @@ -25,7 +25,10 @@ def birefringence_phase_recon_settings_function(tmp_path): phase=settings.PhaseSettings(), ) dataset = open_ome_zarr( - tmp_path, layout="fov", mode="w", channel_names=["None"] + tmp_path, + layout="fov", + mode="w", + channel_names=[f"State{i}" for i in range(4)], ) yield recon_settings, dataset @@ -37,6 +40,9 @@ def fluorescence_recon_settings_function(tmp_path): fluorescence=settings.FluorescenceSettings(), ) dataset = open_ome_zarr( - tmp_path, layout="fov", mode="w", channel_names=["None"] + tmp_path, + layout="fov", + mode="w", + channel_names=[f"State{i}" for i in range(4)], ) yield recon_settings, dataset From 9734847092804d2000d87f33db0c3d0943201751 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:15:01 -0700 Subject: [PATCH 05/13] test on all PRs (#383) --- .github/workflows/test_and_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index 4c037308..89481a23 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -11,7 +11,7 @@ on: - "*" pull_request: branches: - - main + - "*" workflow_dispatch: jobs: From ecd3ebe3352d4f7e8f86a04c4bc470daa99f2d05 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 10:56:54 -0700 Subject: [PATCH 06/13] update `waveorder` dependency --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index c7548797..d79769ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.9 setup_requires = setuptools_scm # add your package requirements here install_requires = - waveorder==2.0.0rc0 + waveorder==2.0.0rc1 pycromanager==0.27.2 click>=8.0.1 natsort>=7.1.1 From f6a22d86c1bd54b9f028a8f4d0fac7054e2357e6 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:19:04 -0700 Subject: [PATCH 07/13] support additional channels in input dataset --- recOrder/cli/apply_inverse_transfer_function.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index e9bbda37..a0742019 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -164,10 +164,10 @@ def apply_inverse_transfer_function_cli( echo_headline("Reconstructing phase...") # check data shapes - if input_dataset.data.shape[1] != 1: - raise ValueError( - "You have requested a phase-only reconstruction, but the input dataset has more than one channel." - ) + if tczyx_data.shape[1] != 1: + raise ValueError( + "You have requested a phase-only reconstruction, but the input dataset has more than one channel." + ) # [phase only, 2] if recon_dim == 2: @@ -385,10 +385,10 @@ def apply_inv_tf( """ Apply an inverse transfer function to a dataset using a configuration file. - See /examples/settings/ for example configuration files. + See /examples for example configuration files. Example usage:\n - $ recorder apply-inv-tf input.zarr/0/0/0 transfer-function.zarr -c /examples/settings/birefringence.yml -o output.zarr + $ recorder apply-inv-tf input.zarr/0/0/0 transfer-function.zarr -c /examples/birefringence.yml -o output.zarr """ apply_inverse_transfer_function_cli( input_data_path, transfer_function_path, config_path, output_path From 623a4991d9d86a59eb3ddfb11e87d3833edb00a3 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:19:15 -0700 Subject: [PATCH 08/13] Update docs to example settings --- recOrder/cli/compute_transfer_function.py | 4 ++-- recOrder/cli/reconstruct.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/recOrder/cli/compute_transfer_function.py b/recOrder/cli/compute_transfer_function.py index 729531ff..80b740aa 100644 --- a/recOrder/cli/compute_transfer_function.py +++ b/recOrder/cli/compute_transfer_function.py @@ -211,9 +211,9 @@ def compute_tf(input_data_path, config_path, output_path): """ Compute a transfer function using a dataset and configuration file. - See /examples/settings/ for example configuration files. + See /examples/ for example configuration files. Example usage:\n - $ recorder compute-tf input.zarr/0/0/0 -c /examples/settings/birefringence.yml -o transfer_function.zarr + $ recorder compute-tf input.zarr/0/0/0 -c /examples/birefringence.yml -o transfer_function.zarr """ compute_transfer_function_cli(input_data_path, config_path, output_path) diff --git a/recOrder/cli/reconstruct.py b/recOrder/cli/reconstruct.py index 9844929d..8599a675 100644 --- a/recOrder/cli/reconstruct.py +++ b/recOrder/cli/reconstruct.py @@ -23,10 +23,10 @@ def reconstruct(input_data_path, config_path, output_path): convenience function for a `compute-tf` call followed by a `apply-inv-tf` call. - See /examples/settings/ for example configuration files. + See /examples for example configuration files. Example usage:\n - $ recorder reconstruct input.zarr/0/0/0 -c /examples/settings/birefringence.yml -o output.zarr + $ recorder reconstruct input.zarr/0/0/0 -c /examples/birefringence.yml -o output.zarr """ # Handle transfer function path From 2e974a38607da1c79ce8eefe50096941088a1fba Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:51:25 -0700 Subject: [PATCH 09/13] convert from np.uint16 to np.int32 then torch.float32 --- recOrder/cli/apply_inverse_transfer_function.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index a0742019..45fe088b 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -109,9 +109,10 @@ def apply_inverse_transfer_function_cli( ) # Load data + tczyx_uint16_numpy = input_dataset.data.oindex[:, channel_indices] tczyx_data = torch.tensor( - input_dataset.data.oindex[:, channel_indices], dtype=torch.float32 - ) + np.int32(tczyx_uint16_numpy), dtype=torch.float32 + ) # convert to np.int32 (torch doesn't accept np.uint16), then convert to tensor float32 # Prepare background dataset if settings.birefringence is not None: From 9fdb6b754ad6a3c7f09ce1f061d410f8c2baa315 Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:51:52 -0700 Subject: [PATCH 10/13] np.uint16 to mimic MM datasets --- recOrder/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/tests/conftest.py b/recOrder/tests/conftest.py index 28148d0c..a6bd4275 100644 --- a/recOrder/tests/conftest.py +++ b/recOrder/tests/conftest.py @@ -14,7 +14,7 @@ def input_zarr(tmp_path): mode="w", channel_names=[f"State{i}" for i in range(4)] + ["BF"], ) - dataset.create_zeros("0", (2, 5, 4, 5, 6), dtype=np.float32) + dataset.create_zeros("0", (2, 5, 4, 5, 6), dtype=np.uint16) yield path, dataset From 84b74806ffe8963500021013134d601d58a614bc Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:52:44 -0700 Subject: [PATCH 11/13] formatting fix --- recOrder/cli/apply_inverse_transfer_function.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index 45fe088b..46cac81a 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -166,9 +166,9 @@ def apply_inverse_transfer_function_cli( # check data shapes if tczyx_data.shape[1] != 1: - raise ValueError( - "You have requested a phase-only reconstruction, but the input dataset has more than one channel." - ) + raise ValueError( + "You have requested a phase-only reconstruction, but the input dataset has more than one channel." + ) # [phase only, 2] if recon_dim == 2: From 8fdf5d060127646a5e1986c7cb161721f895903f Mon Sep 17 00:00:00 2001 From: Talon Chandler Date: Tue, 25 Jul 2023 11:53:33 -0700 Subject: [PATCH 12/13] make stack traces visible when tests fail --- recOrder/tests/cli_tests/test_reconstruct.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recOrder/tests/cli_tests/test_reconstruct.py b/recOrder/tests/cli_tests/test_reconstruct.py index e6f584ca..ce1d58ac 100644 --- a/recOrder/tests/cli_tests/test_reconstruct.py +++ b/recOrder/tests/cli_tests/test_reconstruct.py @@ -55,6 +55,7 @@ def test_reconstruct(tmp_path): + str(config_path) + " -o " + str(tf_path), + catch_exceptions=False, ) assert tf_path.exists() @@ -71,6 +72,7 @@ def test_reconstruct(tmp_path): + str(config_path) + " -o " + str(result_path), + catch_exceptions=False, ) assert result_path.exists() assert result_inv.exit_code == 0 From 85b82851d6e2faad6509642dd8cc867f0e2c7c84 Mon Sep 17 00:00:00 2001 From: Ziwen Liu <67518483+ziw-liu@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:40:39 -0700 Subject: [PATCH 13/13] replace string concatenation with list of args --- recOrder/tests/cli_tests/test_reconstruct.py | 33 ++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/recOrder/tests/cli_tests/test_reconstruct.py b/recOrder/tests/cli_tests/test_reconstruct.py index ce1d58ac..cdf4f8c8 100644 --- a/recOrder/tests/cli_tests/test_reconstruct.py +++ b/recOrder/tests/cli_tests/test_reconstruct.py @@ -49,13 +49,14 @@ def test_reconstruct(tmp_path): tf_path = input_path.with_name("tf.zarr") runner.invoke( cli, - "compute-tf " - + str(input_path) - + " -c " - + str(config_path) - + " -o " - + str(tf_path), - catch_exceptions=False, + [ + "compute-tf", + str(input_path), + "-c", + str(config_path), + "-o", + str(tf_path), + ], ) assert tf_path.exists() @@ -64,15 +65,15 @@ def test_reconstruct(tmp_path): result_inv = runner.invoke( cli, - "apply-inv-tf " - + str(input_path) - + " " - + str(tf_path) - + " -c " - + str(config_path) - + " -o " - + str(result_path), - catch_exceptions=False, + [ + "apply-inv-tf", + str(input_path), + str(tf_path), + "-c", + str(config_path), + "-o", + str(result_path), + ], ) assert result_path.exists() assert result_inv.exit_code == 0