update the tokenizers version (#1112)

* update the tokenizers version * Update optimum/intel/utils/import_utils.py Co-authored-by: Ekaterina Aidova <[email protected]> --------- Co-authored-by: Alexander Kozlov <[email protected]> Co-authored-by: Ekaterina Aidova <[email protected]>
huggingface · Jan 16, 2025 · feaf027 · feaf027
1 parent a11c6c8
commit feaf027
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
@@ -397,6 +397,7 @@ def is_openvino_tokenizers_version(operation: str, version: str):
         except importlib_metadata.PackageNotFoundError:
             pass
 
+    tokenizers_version = tokenizers_version[: len("2025.0.0.0")]
     return compare_versions(parse(tokenizers_version), operation, version)