From fd1a64641f0e291b87529a69ceb5aa50ec2e3d9d Mon Sep 17 00:00:00 2001 From: pablodanswer Date: Thu, 14 Nov 2024 11:53:12 -0800 Subject: [PATCH] minor cosmetic update --- .../natural_language_processing/utils.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/backend/danswer/natural_language_processing/utils.py b/backend/danswer/natural_language_processing/utils.py index f3ef2f929c2..5c80177689d 100644 --- a/backend/danswer/natural_language_processing/utils.py +++ b/backend/danswer/natural_language_processing/utils.py @@ -130,17 +130,15 @@ def _check_tokenizer_cache( def get_tokenizer( model_name: str | None, provider_type: EmbeddingProvider | str | None ) -> BaseTokenizer: - if provider_type is not None: - if isinstance(provider_type, str): - try: - provider_type = EmbeddingProvider(provider_type) - except ValueError: - logger.debug( - f"Invalid provider_type '{provider_type}'. Falling back to default tokenizer." - ) - return _DEFAULT_TOKENIZER - return _check_tokenizer_cache(provider_type, model_name) - return _DEFAULT_TOKENIZER + if isinstance(provider_type, str): + try: + provider_type = EmbeddingProvider(provider_type) + except ValueError: + logger.debug( + f"Invalid provider_type '{provider_type}'. Falling back to default tokenizer." + ) + return _DEFAULT_TOKENIZER + return _check_tokenizer_cache(provider_type, model_name) def tokenizer_trim_content(