From efa06f8ad9aa14b5e5d712f24582f042790dc6e3 Mon Sep 17 00:00:00 2001 From: pablodanswer Date: Thu, 14 Nov 2024 17:00:03 -0800 Subject: [PATCH] nit --- backend/danswer/natural_language_processing/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/danswer/natural_language_processing/utils.py b/backend/danswer/natural_language_processing/utils.py index 5c4bb7d5836..56631495a4e 100644 --- a/backend/danswer/natural_language_processing/utils.py +++ b/backend/danswer/natural_language_processing/utils.py @@ -111,6 +111,8 @@ def _check_tokenizer_cache( def _try_initialize_tokenizer( model_name: str, model_provider: EmbeddingProvider | None ) -> BaseTokenizer | None: + tokenizer: BaseTokenizer | None = None + if model_provider is not None: # Try using TiktokenTokenizer first if model_provider exists try: