update version tag, minor adjustment to AutoTokenizer ctor in example…

… to accommodate HF option change
speediedan · Aug 15, 2024 · ea7fef7 · ea7fef7
1 parent 2e7be86
commit ea7fef7
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 13 deletions.
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,4 +1,4 @@
-#lightning>=2.4.0,<2.4.1
+lightning>=2.4.0,<2.4.1
 # the below is uncommented when master is targeting a specific pl dev master commit
-git+https://github.com/Lightning-AI/lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=lightning
+#git+https://github.com/Lightning-AI/lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=lightning
 torch>=2.1.0
diff --git a/requirements/standalone_base.txt b/requirements/standalone_base.txt
@@ -1,4 +1,4 @@
-#pytorch-lightning>=2.4.0,<2.4.1
+pytorch-lightning>=2.4.0,<2.4.1
 # the below is uncommented when master is targeting a specific pl dev master commit
-git+https://github.com/Lightning-AI/pytorch-lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=pytorch-lightning
+#git+https://github.com/Lightning-AI/pytorch-lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=pytorch-lightning
 torch>=2.1.0
diff --git a/setup.py b/setup.py
@@ -128,15 +128,15 @@ def _setup_args(standalone: bool = False) -> Dict[str, Any]:
     )
 
     base_reqs = "standalone_base.txt" if standalone else "base.txt"
-    # install_requires = setup_tools._load_requirements(
-    #     _INSTALL_PATHS["require"], file_name=base_reqs, standalone=standalone
-    # )
     install_requires = setup_tools._load_requirements(
-        _INSTALL_PATHS["require"],
-        file_name=base_reqs,
-        standalone=standalone,
-        pl_commit="2064887b12dd934a5f9a2bf45897f29e3bfc74d1",
+        _INSTALL_PATHS["require"], file_name=base_reqs, standalone=standalone
     )
+    # install_requires = setup_tools._load_requirements(
+    #     _INSTALL_PATHS["require"],
+    #     file_name=base_reqs,
+    #     standalone=standalone,
+    #     pl_commit="2064887b12dd934a5f9a2bf45897f29e3bfc74d1",
+    # )
     base_setup["install_requires"] = install_requires
     return base_setup
 

diff --git a/src/finetuning_scheduler/__about__.py b/src/finetuning_scheduler/__about__.py
@@ -1,7 +1,7 @@
 import time
 
 _this_year = time.strftime("%Y")
-__version__ = "2.4.0.dev0"
+__version__ = "2.4.0"
 __author__ = "Dan Dale"
 __author_email__ = "[email protected]"
 __license__ = "Apache-2.0"

diff --git a/src/fts_examples/stable/fts_superglue.py b/src/fts_examples/stable/fts_superglue.py
@@ -135,6 +135,7 @@ def __init__(
         }
         os.environ["TOKENIZERS_PARALLELISM"] = "true" if self.hparams.tokenizers_parallelism else "false"
         self.tokenizer = AutoTokenizer.from_pretrained(self.hparams.model_name_or_path, use_fast=True,
+                                                       clean_up_tokenization_spaces=True,
                                                        local_files_only=False)
 
     def prepare_data(self):

diff --git a/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py b/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py
@@ -262,7 +262,7 @@ def __init__(
         self.save_hyperparameters()
         os.environ["TOKENIZERS_PARALLELISM"] = "true" if self.hparams.tokenizers_parallelism else "false"
         self.tokenizer = AutoTokenizer.from_pretrained(
-            self.hparams.model_name_or_path, use_fast=True, local_files_only=False
+            self.hparams.model_name_or_path, use_fast=True, local_files_only=False, clean_up_tokenization_spaces=True,
         )
 
     def prepare_data(self):