FIX: add line for check duplicates_save_path is None

UpstageAI · May 9, 2024 · a0adedc · a0adedc
1 parent 757e87e
commit a0adedc
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/dataverse/etl/deduplication/minhash.py b/dataverse/etl/deduplication/minhash.py
@@ -215,7 +215,10 @@ def deduplication___minhash___lsh_jaccard(
     elif isinstance(data, DataFrame):
         data_df = data
 
-    if os.path.exists(duplicates_save_path):
+    if (
+        duplicates_save_path is not None 
+        and os.path.exists(duplicates_save_path)
+    ):
         assert "duplicates_save_path already exists."
 
     temp_id_col, component_col, tokens_col, ngrams_col = \