From ea7fef73b99c781df8e4040ba8603265b67488c8 Mon Sep 17 00:00:00 2001 From: Daniel Dale Date: Thu, 15 Aug 2024 08:36:39 -0700 Subject: [PATCH] update version tag, minor adjustment to AutoTokenizer ctor in example to accommodate HF option change --- requirements/base.txt | 4 ++-- requirements/standalone_base.txt | 4 ++-- setup.py | 14 +++++++------- src/finetuning_scheduler/__about__.py | 2 +- src/fts_examples/stable/fts_superglue.py | 1 + .../stable/ipynb_src/fts_superglue_nb.py | 2 +- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index f366b4a..7b733d7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -#lightning>=2.4.0,<2.4.1 +lightning>=2.4.0,<2.4.1 # the below is uncommented when master is targeting a specific pl dev master commit -git+https://github.com/Lightning-AI/lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=lightning +#git+https://github.com/Lightning-AI/lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=lightning torch>=2.1.0 diff --git a/requirements/standalone_base.txt b/requirements/standalone_base.txt index 062921f..2db7086 100644 --- a/requirements/standalone_base.txt +++ b/requirements/standalone_base.txt @@ -1,4 +1,4 @@ -#pytorch-lightning>=2.4.0,<2.4.1 +pytorch-lightning>=2.4.0,<2.4.1 # the below is uncommented when master is targeting a specific pl dev master commit -git+https://github.com/Lightning-AI/pytorch-lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=pytorch-lightning +#git+https://github.com/Lightning-AI/pytorch-lightning.git@2064887b12dd934a5f9a2bf45897f29e3bfc74d1#egg=pytorch-lightning torch>=2.1.0 diff --git a/setup.py b/setup.py index 682a445..499b01e 100755 --- a/setup.py +++ b/setup.py @@ -128,15 +128,15 @@ def _setup_args(standalone: bool = False) -> Dict[str, Any]: ) base_reqs = "standalone_base.txt" if standalone else "base.txt" - # install_requires = setup_tools._load_requirements( - # _INSTALL_PATHS["require"], file_name=base_reqs, standalone=standalone - # ) install_requires = setup_tools._load_requirements( - _INSTALL_PATHS["require"], - file_name=base_reqs, - standalone=standalone, - pl_commit="2064887b12dd934a5f9a2bf45897f29e3bfc74d1", + _INSTALL_PATHS["require"], file_name=base_reqs, standalone=standalone ) + # install_requires = setup_tools._load_requirements( + # _INSTALL_PATHS["require"], + # file_name=base_reqs, + # standalone=standalone, + # pl_commit="2064887b12dd934a5f9a2bf45897f29e3bfc74d1", + # ) base_setup["install_requires"] = install_requires return base_setup diff --git a/src/finetuning_scheduler/__about__.py b/src/finetuning_scheduler/__about__.py index 4b05df4..7a372d6 100644 --- a/src/finetuning_scheduler/__about__.py +++ b/src/finetuning_scheduler/__about__.py @@ -1,7 +1,7 @@ import time _this_year = time.strftime("%Y") -__version__ = "2.4.0.dev0" +__version__ = "2.4.0" __author__ = "Dan Dale" __author_email__ = "danny.dale@gmail.com" __license__ = "Apache-2.0" diff --git a/src/fts_examples/stable/fts_superglue.py b/src/fts_examples/stable/fts_superglue.py index e9b1c41..8b02e51 100644 --- a/src/fts_examples/stable/fts_superglue.py +++ b/src/fts_examples/stable/fts_superglue.py @@ -135,6 +135,7 @@ def __init__( } os.environ["TOKENIZERS_PARALLELISM"] = "true" if self.hparams.tokenizers_parallelism else "false" self.tokenizer = AutoTokenizer.from_pretrained(self.hparams.model_name_or_path, use_fast=True, + clean_up_tokenization_spaces=True, local_files_only=False) def prepare_data(self): diff --git a/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py b/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py index efc2b81..66ba725 100644 --- a/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py +++ b/src/fts_examples/stable/ipynb_src/fts_superglue_nb.py @@ -262,7 +262,7 @@ def __init__( self.save_hyperparameters() os.environ["TOKENIZERS_PARALLELISM"] = "true" if self.hparams.tokenizers_parallelism else "false" self.tokenizer = AutoTokenizer.from_pretrained( - self.hparams.model_name_or_path, use_fast=True, local_files_only=False + self.hparams.model_name_or_path, use_fast=True, local_files_only=False, clean_up_tokenization_spaces=True, ) def prepare_data(self):