From 4b98d7438f177ce47d277f57661ecc9dc071a3a3 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Thu, 31 Oct 2024 14:25:56 -0500
Subject: [PATCH 01/21] adding yml file for conda environment

---
 environment.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 environment.yml

diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000..e69de29

From 026afed7a41797708d2cb2ad2149b0dc6f7ba154 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Thu, 14 Nov 2024 13:57:50 -0600
Subject: [PATCH 02/21] changes and adding new config for planet data

---
 config.toml        | 2 ++
 planet_config.toml | 7 +++++++
 2 files changed, 9 insertions(+)
 create mode 100644 planet_config.toml

diff --git a/config.toml b/config.toml
index af857dc..cda9a01 100644
--- a/config.toml
+++ b/config.toml
@@ -2,6 +2,8 @@
 [darts]
 ee-project = "your-project-here"
 
+
+
 [darts.paths]
 sentinel2-dir = "data/input/sentinel2"
 orthotiles-dir = "data/input/planet/PSOrthoTile"
diff --git a/planet_config.toml b/planet_config.toml
new file mode 100644
index 0000000..9a7196b
--- /dev/null
+++ b/planet_config.toml
@@ -0,0 +1,7 @@
+[darts]
+orthotiles-dir = "data/input/planet/PSOrthoTile"
+scenes-dir = "data/input/planet/PSScene"
+arcticdem-slope-vrt = "data/input/ArcticDEM/slope.vrt"
+arcticdem-elevation-vrt = "data/input/ArcticDEM/elevation.vrt"
+output-data-dir = "data/output"
+model-dir = "models"
\ No newline at end of file

From 34b369483241385db0ae2fb9b9edf9067636b544 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 25 Nov 2024 10:31:19 -0600
Subject: [PATCH 03/21] start of docs for conda install

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 97349cc..7fe9ded 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,14 @@ At the first start, VSCode should ask you if you want to install the recommended
 The settings should be automaticly used by VSCode.
 Both should provide the developers with a better experience and enforce code-style.
 
+
+## Conda Environment setup
+
+Prereq:
+ - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
+ - Begin by creating a conda environment
+
+
 ## Environment setup
 
 Prereq:

From 1a28d3208cd02fc509d93aa14774ccfb825b4e05 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 25 Nov 2024 10:43:19 -0600
Subject: [PATCH 04/21] more instructions for conda installation

---
 README.md | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7fe9ded..b4a66c3 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,16 @@ Both should provide the developers with a better experience and enforce code-sty
 
 Prereq:
  - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
- - Begin by creating a conda environment
+ - Begin by creating a conda environment `conda create -n darts-nextgen python=3.11`
+ - Install dependencies from project.toml `pip install pyproject.toml`
+ - Install dependencies for darts-acquisition `cd darts-acquisition` then `pip install pyproject.toml`
+ - Install dependencies for darts-ensemble `cd darts-ensemble` then `pip install pyproject.toml`
+ - Install dependencies for darts-export `cd darts-export` then `pip install pyproject.toml`
+ - Install dependencies for darts-postprocessing `cd darts-postprocessing` then `pip install pyproject.toml`
+ - Install dependencies for darts-preprocessing `cd darts-preprocessing` then `pip install pyproject.toml`
+ - Install dependencies for darts-superresolution `cd darts-superresolution` then `pip install pyproject.toml`
+ - Install dependencies for darts-segmentation `cd darts-segmentation` then `pip install pyproject.toml`
+
 
 
 ## Environment setup

From adefafff93340aa64086d6722c73724c1dfd0356 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 25 Nov 2024 12:13:10 -0600
Subject: [PATCH 05/21] new better instructios for installing dependencies
 using conda env

---
 README.md | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/README.md b/README.md
index b4a66c3..4d80091 100644
--- a/README.md
+++ b/README.md
@@ -23,14 +23,7 @@ Both should provide the developers with a better experience and enforce code-sty
 Prereq:
  - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
  - Begin by creating a conda environment `conda create -n darts-nextgen python=3.11`
- - Install dependencies from project.toml `pip install pyproject.toml`
- - Install dependencies for darts-acquisition `cd darts-acquisition` then `pip install pyproject.toml`
- - Install dependencies for darts-ensemble `cd darts-ensemble` then `pip install pyproject.toml`
- - Install dependencies for darts-export `cd darts-export` then `pip install pyproject.toml`
- - Install dependencies for darts-postprocessing `cd darts-postprocessing` then `pip install pyproject.toml`
- - Install dependencies for darts-preprocessing `cd darts-preprocessing` then `pip install pyproject.toml`
- - Install dependencies for darts-superresolution `cd darts-superresolution` then `pip install pyproject.toml`
- - Install dependencies for darts-segmentation `cd darts-segmentation` then `pip install pyproject.toml`
+ - Install dependencies from project.toml `pip install '.[dev]'`
 
 
 

From c70985d5e7407b0fe29a90b317012f86a2658c13 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 25 Nov 2024 16:09:18 -0600
Subject: [PATCH 06/21] set cyclopts version to 2.9.9

---
 README.md      | 2 +-
 pyproject.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4d80091..17e8256 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ Prereq:
  - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
  - Begin by creating a conda environment `conda create -n darts-nextgen python=3.11`
  - Install dependencies from project.toml `pip install '.[dev]'`
-
+ - Do the same in other packages.
 
 
 ## Environment setup
diff --git a/pyproject.toml b/pyproject.toml
index a129b17..a77fd89 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ dependencies = [
     "xarray>=2024.9.0",
     "ray[data]>=2.37.0",
     "rich>=13.9.2",
-    "cyclopts>=2.9.9",
+    "cyclopts==2.9.9",
     # Utils
     "lovely-tensors>=0.1.17",
     "lovely-numpy>=0.2.13",

From 77dc1ca8965905cef337996d67e92896f2983234 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 25 Nov 2024 16:35:35 -0600
Subject: [PATCH 07/21] mps for mac

---
 darts-segmentation/src/darts_segmentation/segment.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/darts-segmentation/src/darts_segmentation/segment.py b/darts-segmentation/src/darts_segmentation/segment.py
index ee45070..2c48eb6 100644
--- a/darts-segmentation/src/darts_segmentation/segment.py
+++ b/darts-segmentation/src/darts_segmentation/segment.py
@@ -59,7 +59,9 @@ def __init__(self, model_checkpoint: Path | str):
             model_checkpoint (Path): The path to the model checkpoint.
 
         """
+        # TODO better way to check device here
         self.device = torch.device("cpu") if not torch.cuda.is_available() else torch.device("cuda")
+        self.device = torch.device("cpu") if not torch.backends.mps.is_available() else torch.device("mps")
         ckpt = torch.load(model_checkpoint, map_location=self.device)
         self.config = validate_config(ckpt["config"])
         self.model = smp.create_model(**self.config["model"], encoder_weights=None)

From 3818501c5c3f3bd491080c77c76200bb28d94255 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Tue, 26 Nov 2024 13:30:19 -0600
Subject: [PATCH 08/21] adding a python class for running the pipeline for
 easier debugging

---
 README.md                     |  2 +-
 darts/src/darts/run_native.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 darts/src/darts/run_native.py

diff --git a/README.md b/README.md
index 17e8256..b501672 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ Prereq:
  - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
  - Begin by creating a conda environment `conda create -n darts-nextgen python=3.11`
  - Install dependencies from project.toml `pip install '.[dev]'`
- - Do the same in other packages.
+ - Do the same in other packages (darts-acquisition to darts-superresolution)
 
 
 ## Environment setup
diff --git a/darts/src/darts/run_native.py b/darts/src/darts/run_native.py
new file mode 100644
index 0000000..52d0b8b
--- /dev/null
+++ b/darts/src/darts/run_native.py
@@ -0,0 +1,17 @@
+from darts.native import run_native_planet_pipeline
+from pathlib import Path
+orthotiles_dir = "data/input/planet/PSOrthoTile"
+scenes_dir = "data/input/planet/PSScene"
+arcticdem_slope_vrt = "data/input/ArcticDEM/slope.vrt"
+arcticdem_elevation_vrt = "data/input/ArcticDEM/elevation.vrt"
+output_data_dir = "data/output"
+model_dir = "models"
+
+if __name__ == "__main__":
+    print("Running the pipeline")
+    run_native_planet_pipeline(orthotiles_dir=Path(orthotiles_dir),
+                               scenes_dir=Path(scenes_dir),
+                               output_data_dir=Path(output_data_dir),
+                               arcticdem_slope_vrt=Path(arcticdem_slope_vrt),
+                               arcticdem_elevation_vrt=Path(arcticdem_elevation_vrt),
+                               model_dir=Path(model_dir))
\ No newline at end of file

From f940bd5b7bc5798d9266f3a2779fc1cc57b46dd5 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Fri, 29 Nov 2024 12:14:37 -0600
Subject: [PATCH 09/21] trying to use metal for mac os

---
 darts/src/darts/utils/cuda.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/darts/src/darts/utils/cuda.py b/darts/src/darts/utils/cuda.py
index 920ff89..a094ec4 100644
--- a/darts/src/darts/utils/cuda.py
+++ b/darts/src/darts/utils/cuda.py
@@ -73,7 +73,12 @@ def decide_device(device: Literal["cuda", "cpu", "auto"] | int | None) -> Litera
     # We can't provide a default value for device in the parameter list because then we would need to import torch at
     # top-level, which would make the CLI slow.
     if device is None:
-        device = "cuda" if torch.cuda.is_available() and has_cuda_and_cupy() else "cpu"
+        if torch.cuda.is_available() and has_cuda_and_cupy():
+            device = "cuda"
+        elif torch.backends.mps.is_available():
+            device = "mps"
+        else:
+            device = "cpu"
         logger.info(f"Device not provided. Using {device}.")
         return device
 

From b973c6fd1692d225a6496f90da0456b64d4f0b16 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Fri, 29 Nov 2024 12:24:35 -0600
Subject: [PATCH 10/21] error now due to nvml library need to fix to work with
 mac

---
 darts/src/darts/run_native.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/darts/src/darts/run_native.py b/darts/src/darts/run_native.py
index 52d0b8b..3e5133e 100644
--- a/darts/src/darts/run_native.py
+++ b/darts/src/darts/run_native.py
@@ -7,6 +7,8 @@
 output_data_dir = "data/output"
 model_dir = "models"
 
+tcvis_dir = "tcvis_dir"
+
 if __name__ == "__main__":
     print("Running the pipeline")
     run_native_planet_pipeline(orthotiles_dir=Path(orthotiles_dir),
@@ -14,4 +16,5 @@
                                output_data_dir=Path(output_data_dir),
                                arcticdem_slope_vrt=Path(arcticdem_slope_vrt),
                                arcticdem_elevation_vrt=Path(arcticdem_elevation_vrt),
-                               model_dir=Path(model_dir))
\ No newline at end of file
+                               model_dir=Path(model_dir),
+                               tcvis_dir=Path(tcvis_dir))
\ No newline at end of file

From 7cd4bf0214ee69f228ef317a7c40edca8af4d218 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Mon, 2 Dec 2024 12:06:36 -0600
Subject: [PATCH 11/21] on sd-gpu this version is needed for cyclopts

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 99b0035..87c4210 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ dependencies = [
     "xarray>=2024.9.0",
     "ray[data]>=2.37.0",
     "rich>=13.9.2",
-    "cyclopts==2.9.9",
+    "cyclopts==3.1.2",
     # Utils
     "lovely-tensors>=0.1.17",
     "lovely-numpy>=0.2.13",

From fd5f64789d6c64955943a612dfd055c8b109b32c Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Mon, 2 Dec 2024 14:42:32 -0600
Subject: [PATCH 12/21] add project name

---
 darts/src/darts/run_native.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/darts/src/darts/run_native.py b/darts/src/darts/run_native.py
index 3e5133e..fd715af 100644
--- a/darts/src/darts/run_native.py
+++ b/darts/src/darts/run_native.py
@@ -17,4 +17,5 @@
                                arcticdem_slope_vrt=Path(arcticdem_slope_vrt),
                                arcticdem_elevation_vrt=Path(arcticdem_elevation_vrt),
                                model_dir=Path(model_dir),
-                               tcvis_dir=Path(tcvis_dir))
\ No newline at end of file
+                               tcvis_dir=Path(tcvis_dir),
+                               ee_project='pdg-project-406720')
\ No newline at end of file

From 76ddf1f878ceb41cb65b8e559f9204c33ae88157 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Mon, 2 Dec 2024 14:53:40 -0600
Subject: [PATCH 13/21] full paths for directories in run_native

---
 darts/src/darts/run_native.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/darts/src/darts/run_native.py b/darts/src/darts/run_native.py
index fd715af..8b1b907 100644
--- a/darts/src/darts/run_native.py
+++ b/darts/src/darts/run_native.py
@@ -1,13 +1,13 @@
 from darts.native import run_native_planet_pipeline
 from pathlib import Path
-orthotiles_dir = "data/input/planet/PSOrthoTile"
-scenes_dir = "data/input/planet/PSScene"
-arcticdem_slope_vrt = "data/input/ArcticDEM/slope.vrt"
-arcticdem_elevation_vrt = "data/input/ArcticDEM/elevation.vrt"
-output_data_dir = "data/output"
-model_dir = "models"
+orthotiles_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSOrthoTile"
+scenes_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSScene"
+arcticdem_slope_vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/slope.vrt"
+arcticdem_elevation_vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/elevation.vrt"
+output_data_dir = "/home/toddn/pdg/darts-nextgen/data/output"
+model_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/models"
 
-tcvis_dir = "tcvis_dir"
+tcvis_dir = "/home/toddn/pdg/darts-nextgen/tcvis_dir"
 
 if __name__ == "__main__":
     print("Running the pipeline")

From f998520ea866930ef18fed7db59c6d40f19e55bc Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Mon, 2 Dec 2024 15:24:01 -0600
Subject: [PATCH 14/21] full paths for directories in run_native specify
 project tcvis dir

---
 planet_config.toml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/planet_config.toml b/planet_config.toml
index 9a7196b..33089b4 100644
--- a/planet_config.toml
+++ b/planet_config.toml
@@ -1,7 +1,9 @@
 [darts]
-orthotiles-dir = "data/input/planet/PSOrthoTile"
-scenes-dir = "data/input/planet/PSScene"
-arcticdem-slope-vrt = "data/input/ArcticDEM/slope.vrt"
-arcticdem-elevation-vrt = "data/input/ArcticDEM/elevation.vrt"
-output-data-dir = "data/output"
-model-dir = "models"
\ No newline at end of file
+orthotiles-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSOrthoTile"
+scenes-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSScene"
+arcticdem-slope-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/slope.vrt"
+arcticdem-elevation-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/elevation.vrt"
+output-data-dir = "/home/toddn/pdg/darts-nextgen/data/output"
+model-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/models"
+tcvis-dir= "/home/toddn/pdg/darts-nextgen/tcvis_dir"
+ee-project = "pdg-project-406720"
\ No newline at end of file

From 2ce6527b87e8e72f0e0881104cb23972e3f46ea7 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Tue, 3 Dec 2024 15:26:51 -0600
Subject: [PATCH 15/21] script creates and re-installs the conda environment if
 it already existed

---
 create_env.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100755 create_env.sh

diff --git a/create_env.sh b/create_env.sh
new file mode 100755
index 0000000..0301f3d
--- /dev/null
+++ b/create_env.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+CONDA_ENVS="$(conda env list)"
+current_directory=$(pwd)
+echo "${current_directory}"
+
+echo "${CONDA_ENVS}"
+
+conda remove -n darts-nextgen-test -all -y
+
+conda create -n darts-nextgen-test python=3.11 -y
+
+conda activate darts-nextgen-test
+
+CUR_PYTHON="$(which python)"
+
+ pip install '.[dev]'
+
+
+cd ${current_directory}/darts-acquisition
+new_dir=$(pwd)
+echo "${new_dir}"
+pip install '.[dev]'
+
+cd ${current_directory}/darts-ensemble
+pip install '.[dev]'
+
+cd ${current_directory}/darts-export
+pip install '.[dev]'
+
+cd ${current_directory}/darts-postprocessing
+pip install '.[dev]'
+
+cd ${current_directory}/darts-preprocessing
+pip install '.[dev]'
+
+cd ${current_directory}/darts-segmentation
+pip install '.[dev]'
+
+
+cd ${current_directory}/darts-superresolution
+pip install '.[dev]'
+
+
+cd ${current_directory}/darts-utils
+pip install '.[dev]'
+
+cd ${current_directory}
+
+echo Created Environment
\ No newline at end of file

From b736712624281d2dd70cc572cf5d78f1c664005e Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Tue, 3 Dec 2024 15:31:28 -0600
Subject: [PATCH 16/21] better instructions

---
 README.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b501672..86e1553 100644
--- a/README.md
+++ b/README.md
@@ -17,8 +17,18 @@ At the first start, VSCode should ask you if you want to install the recommended
 The settings should be automaticly used by VSCode.
 Both should provide the developers with a better experience and enforce code-style.
 
+## Create Conda Environment from Script
 
-## Conda Environment setup
+This script will create (or recreate) the conda environment from a bash script.
+
+Prereq
+ - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda
+ - Install by running this command `. create_env.sh`
+ - The conda environment is installed, activate it and type `darts --help`
+
+## Manual Conda Environment setup
+
+If you prefer to install conda manually, here are steps
 
 Prereq:
  - [Conda] (https://docs.anaconda.com/miniconda/): Link to miniconda. Can use any other conda

From d5bc3173b8b2740d064eb99fd19824aa6f53b985 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Tue, 3 Dec 2024 17:30:00 -0600
Subject: [PATCH 17/21] fixes error, not sure if this is a good policy, but
 works enough

---
 darts-preprocessing/src/darts_preprocessing/preprocess.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/darts-preprocessing/src/darts_preprocessing/preprocess.py b/darts-preprocessing/src/darts_preprocessing/preprocess.py
index 76a9504..cb5c3e5 100644
--- a/darts-preprocessing/src/darts_preprocessing/preprocess.py
+++ b/darts-preprocessing/src/darts_preprocessing/preprocess.py
@@ -31,6 +31,7 @@ def preprocess_legacy(
     ds_arcticdem: xr.Dataset,
     ds_tcvis: xr.Dataset,
     ds_data_masks: xr.Dataset,
+    compat_overridee: bool = True,
 ) -> xr.Dataset:
     """Preprocess optical data with legacy (DARTS v1) preprocessing steps.
 
@@ -58,8 +59,10 @@ def preprocess_legacy(
     # we dont need to calculate them here
 
     # merge to final dataset
-    ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks])
-
+    if compat_overridee:
+        ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks], compat='override')
+    else:
+        ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks])
     return ds_merged
 
 

From 15aac9c0afff08d35642343f65ea440ba3b10995 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Wed, 4 Dec 2024 09:58:41 -0600
Subject: [PATCH 18/21] reverting change

---
 config.toml.example                                       | 2 --
 darts-preprocessing/src/darts_preprocessing/preprocess.py | 7 ++-----
 pyproject.toml                                            | 2 +-
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/config.toml.example b/config.toml.example
index 0aa76e8..fc1abad 100644
--- a/config.toml.example
+++ b/config.toml.example
@@ -2,8 +2,6 @@
 [darts]
 ee-project = "your-project-here"
 
-
-
 [darts.paths]
 sentinel2-dir = "data/input/sentinel2"
 orthotiles-dir = "data/input/planet/PSOrthoTile"
diff --git a/darts-preprocessing/src/darts_preprocessing/preprocess.py b/darts-preprocessing/src/darts_preprocessing/preprocess.py
index cb5c3e5..76a9504 100644
--- a/darts-preprocessing/src/darts_preprocessing/preprocess.py
+++ b/darts-preprocessing/src/darts_preprocessing/preprocess.py
@@ -31,7 +31,6 @@ def preprocess_legacy(
     ds_arcticdem: xr.Dataset,
     ds_tcvis: xr.Dataset,
     ds_data_masks: xr.Dataset,
-    compat_overridee: bool = True,
 ) -> xr.Dataset:
     """Preprocess optical data with legacy (DARTS v1) preprocessing steps.
 
@@ -59,10 +58,8 @@ def preprocess_legacy(
     # we dont need to calculate them here
 
     # merge to final dataset
-    if compat_overridee:
-        ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks], compat='override')
-    else:
-        ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks])
+    ds_merged = xr.merge([ds_optical, ds_ndvi, ds_arcticdem, ds_tcvis, ds_data_masks])
+
     return ds_merged
 
 
diff --git a/pyproject.toml b/pyproject.toml
index 87c4210..eda1728 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ dependencies = [
     "xarray>=2024.9.0",
     "ray[data]>=2.37.0",
     "rich>=13.9.2",
-    "cyclopts==3.1.2",
+    "cyclopts>=3.1.0",
     # Utils
     "lovely-tensors>=0.1.17",
     "lovely-numpy>=0.2.13",

From d45a02b24fe2605b17adc312861ec43d39aa0bed Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Wed, 1 Jan 2025 14:17:23 -0600
Subject: [PATCH 19/21] adding command to readme

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 89dc167..bd5dec8 100644
--- a/README.md
+++ b/README.md
@@ -148,3 +148,7 @@ logging.basicConfig(
 install(show_locals=True)  # Change to False if you encounter too large tracebacks
 init_ee("ee-project")  # Replace with your project
 ```
+
+currently running planet pipeline using this command 
+
+`darts run-native-planet-pipeline-fast --config-file=planet_config.toml`
\ No newline at end of file

From e8d63bb9c88fe6ca74cc94df45f2fa8b047aa8d8 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Wed, 1 Jan 2025 15:00:01 -0600
Subject: [PATCH 20/21] setup for running fast pipeline

---
 darts/src/darts/run_native.py | 11 ++++++++++-
 planet_config.toml            |  1 +
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/darts/src/darts/run_native.py b/darts/src/darts/run_native.py
index 8b1b907..2f8dcde 100644
--- a/darts/src/darts/run_native.py
+++ b/darts/src/darts/run_native.py
@@ -1,4 +1,4 @@
-from darts.native import run_native_planet_pipeline
+from darts.native import run_native_planet_pipeline, run_native_planet_pipeline_fast
 from pathlib import Path
 orthotiles_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSOrthoTile"
 scenes_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSScene"
@@ -6,11 +6,20 @@
 arcticdem_elevation_vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/elevation.vrt"
 output_data_dir = "/home/toddn/pdg/darts-nextgen/data/output"
 model_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/models"
+arctic_dem_dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM"
 
 tcvis_dir = "/home/toddn/pdg/darts-nextgen/tcvis_dir"
 
 if __name__ == "__main__":
     print("Running the pipeline")
+    run_native_planet_pipeline_fast(orthotiles_dir=Path(orthotiles_dir),
+                                scenes_dir=Path(scenes_dir),
+                                output_data_dir=Path(output_data_dir),
+                                arcticdem_dir=Path(arctic_dem_dir),
+                                model_dir=Path(model_dir),
+                                tcvis_dir=Path(tcvis_dir),
+                                ee_project='pdg-project-406720')
+
     run_native_planet_pipeline(orthotiles_dir=Path(orthotiles_dir),
                                scenes_dir=Path(scenes_dir),
                                output_data_dir=Path(output_data_dir),
diff --git a/planet_config.toml b/planet_config.toml
index 33089b4..7c4817b 100644
--- a/planet_config.toml
+++ b/planet_config.toml
@@ -1,6 +1,7 @@
 [darts]
 orthotiles-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSOrthoTile"
 scenes-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSScene"
+arcticdem-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM"
 arcticdem-slope-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/slope.vrt"
 arcticdem-elevation-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/elevation.vrt"
 output-data-dir = "/home/toddn/pdg/darts-nextgen/data/output"

From 923117b01d2118352943b1f7290ce6d698296b47 Mon Sep 17 00:00:00 2001
From: tcnichol <tcnichol@illinois.edu>
Date: Fri, 3 Jan 2025 14:05:40 -0600
Subject: [PATCH 21/21] fix environment creation

---
 create_env.sh      |  6 +++---
 planet_config.toml | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/create_env.sh b/create_env.sh
index 0301f3d..e7b59c5 100755
--- a/create_env.sh
+++ b/create_env.sh
@@ -6,11 +6,11 @@ echo "${current_directory}"
 
 echo "${CONDA_ENVS}"
 
-conda remove -n darts-nextgen-test -all -y
+conda remove -n darts-nextgen -all -y
 
-conda create -n darts-nextgen-test python=3.11 -y
+conda create -n darts-nextgen python=3.11 -y
 
-conda activate darts-nextgen-test
+conda activate darts-nextgen
 
 CUR_PYTHON="$(which python)"
 
diff --git a/planet_config.toml b/planet_config.toml
index 7c4817b..dedc05d 100644
--- a/planet_config.toml
+++ b/planet_config.toml
@@ -1,10 +1,10 @@
 [darts]
-orthotiles-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSOrthoTile"
-scenes-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/planet/PSScene"
-arcticdem-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM"
-arcticdem-slope-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/slope.vrt"
-arcticdem-elevation-vrt = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/input/ArcticDEM/elevation.vrt"
-output-data-dir = "/home/toddn/pdg/darts-nextgen/data/output"
-model-dir = "/home/toddn/pdg/darts-nextgen/rts_dtaset01/models"
+orthotiles-dir = "/home/toddn/pdg/darts-nextgen/data/input/planet/PSOrthoTile"
+scenes-dir = "/home/toddn/pdg/darts-nextgen/data/input/planet/PSScene"
+arcticdem-dir = "/home/toddn/pdg/darts-nextgen/rts_dataset01/input/ArcticDEM"
+arcticdem-slope-vrt = "/home/toddn/pdg/darts-nextgen/rts_dataset01/input/ArcticDEM/slope.vrt"
+arcticdem-elevation-vrt = "/home/toddn/pdg/darts-nextgen/rts_dataset01/input/ArcticDEM/elevation.vrt"
+output-data-dir = "/home/toddn/pdg/darts-nextgen/rts_dataset01/output"
+model-dir = "/home/toddn/darts-nextgen/rts_dataset01/models"
 tcvis-dir= "/home/toddn/pdg/darts-nextgen/tcvis_dir"
 ee-project = "pdg-project-406720"
\ No newline at end of file