diff --git a/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py b/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py index c2ba10b3875..09c0f7209d4 100644 --- a/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py +++ b/backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py @@ -9,8 +9,8 @@ import sqlalchemy as sa from danswer.db.models import IndexModelStatus -from danswer.search.enums import RecencyBiasSetting -from danswer.search.enums import SearchType +from danswer.context.search.enums import RecencyBiasSetting +from danswer.context.search.enums import SearchType # revision identifiers, used by Alembic. revision = "776b3bbe9092" diff --git a/backend/danswer/chat/chat_utils.py b/backend/danswer/chat/chat_utils.py index f5961010ba5..5e42ae23f5a 100644 --- a/backend/danswer/chat/chat_utils.py +++ b/backend/danswer/chat/chat_utils.py @@ -7,10 +7,10 @@ from danswer.chat.models import CitationInfo from danswer.chat.models import LlmDoc +from danswer.context.search.models import InferenceSection from danswer.db.chat import get_chat_messages_by_session from danswer.db.models import ChatMessage from danswer.llm.answering.models import PreviousMessage -from danswer.search.models import InferenceSection from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/chat/models.py b/backend/danswer/chat/models.py index 159506c0734..3852029c47b 100644 --- a/backend/danswer/chat/models.py +++ b/backend/danswer/chat/models.py @@ -6,10 +6,10 @@ from pydantic import BaseModel from danswer.configs.constants import DocumentSource -from danswer.search.enums import QueryFlow -from danswer.search.enums import SearchType -from danswer.search.models import RetrievalDocs -from danswer.search.models import SearchResponse +from danswer.context.search.enums import QueryFlow +from danswer.context.search.enums import SearchType +from danswer.context.search.models import RetrievalDocs +from danswer.context.search.models import SearchResponse from danswer.tools.tool_implementations.custom.base_tool_types import ToolResultType diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py index 985b6ed196a..9048e21d610 100644 --- a/backend/danswer/chat/process_message.py +++ b/backend/danswer/chat/process_message.py @@ -23,6 +23,16 @@ from danswer.configs.chat_configs import DISABLE_LLM_CHOOSE_SEARCH from danswer.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT from danswer.configs.constants import MessageType +from danswer.context.search.enums import OptionalSearchSetting +from danswer.context.search.enums import QueryFlow +from danswer.context.search.enums import SearchType +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.retrieval.search_runner import inference_sections_from_ids +from danswer.context.search.utils import chunks_or_sections_to_search_docs +from danswer.context.search.utils import dedupe_documents +from danswer.context.search.utils import drop_llm_indices +from danswer.context.search.utils import relevant_sections_to_indices from danswer.db.chat import attach_files_to_chat_message from danswer.db.chat import create_db_search_doc from danswer.db.chat import create_new_chat_message @@ -56,16 +66,6 @@ from danswer.llm.factory import get_main_llm_from_tuple from danswer.llm.utils import litellm_exception_to_error_msg from danswer.natural_language_processing.utils import get_tokenizer -from danswer.search.enums import OptionalSearchSetting -from danswer.search.enums import QueryFlow -from danswer.search.enums import SearchType -from danswer.search.models import InferenceSection -from danswer.search.models import RetrievalDetails -from danswer.search.retrieval.search_runner import inference_sections_from_ids -from danswer.search.utils import chunks_or_sections_to_search_docs -from danswer.search.utils import dedupe_documents -from danswer.search.utils import drop_llm_indices -from danswer.search.utils import relevant_sections_to_indices from danswer.server.query_and_chat.models import ChatMessageDetail from danswer.server.query_and_chat.models import CreateChatMessageRequest from danswer.server.utils import get_json_line diff --git a/backend/danswer/search/__init__.py b/backend/danswer/context/search/__init__.py similarity index 100% rename from backend/danswer/search/__init__.py rename to backend/danswer/context/search/__init__.py diff --git a/backend/danswer/search/enums.py b/backend/danswer/context/search/enums.py similarity index 100% rename from backend/danswer/search/enums.py rename to backend/danswer/context/search/enums.py diff --git a/backend/danswer/search/models.py b/backend/danswer/context/search/models.py similarity index 98% rename from backend/danswer/search/models.py rename to backend/danswer/context/search/models.py index 815fa9d885f..73ef28dc3b6 100644 --- a/backend/danswer/search/models.py +++ b/backend/danswer/context/search/models.py @@ -8,13 +8,13 @@ from danswer.configs.chat_configs import NUM_RETURNED_HITS from danswer.configs.constants import DocumentSource +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import OptionalSearchSetting +from danswer.context.search.enums import SearchType from danswer.db.models import Persona from danswer.db.models import SearchSettings from danswer.indexing.models import BaseChunk from danswer.indexing.models import IndexingSetting -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import OptionalSearchSetting -from danswer.search.enums import SearchType from shared_configs.enums import RerankerProvider diff --git a/backend/danswer/search/pipeline.py b/backend/danswer/context/search/pipeline.py similarity index 93% rename from backend/danswer/search/pipeline.py rename to backend/danswer/context/search/pipeline.py index 183c8729d67..21c518348e7 100644 --- a/backend/danswer/search/pipeline.py +++ b/backend/danswer/context/search/pipeline.py @@ -7,6 +7,22 @@ from danswer.chat.models import SectionRelevancePiece from danswer.configs.chat_configs import DISABLE_LLM_DOC_RELEVANCE +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import QueryFlow +from danswer.context.search.enums import SearchType +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import RerankMetricsContainer +from danswer.context.search.models import RetrievalMetricsContainer +from danswer.context.search.models import SearchQuery +from danswer.context.search.models import SearchRequest +from danswer.context.search.postprocessing.postprocessing import cleanup_chunks +from danswer.context.search.postprocessing.postprocessing import search_postprocessing +from danswer.context.search.preprocessing.preprocessing import retrieval_preprocessing +from danswer.context.search.retrieval.search_runner import retrieve_chunks +from danswer.context.search.utils import inference_section_from_chunks +from danswer.context.search.utils import relevant_sections_to_indices from danswer.db.models import User from danswer.db.search_settings import get_current_search_settings from danswer.document_index.factory import get_default_document_index @@ -16,22 +32,6 @@ from danswer.llm.answering.prune_and_merge import ChunkRange from danswer.llm.answering.prune_and_merge import merge_chunk_intervals from danswer.llm.interfaces import LLM -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import QueryFlow -from danswer.search.enums import SearchType -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceSection -from danswer.search.models import RerankMetricsContainer -from danswer.search.models import RetrievalMetricsContainer -from danswer.search.models import SearchQuery -from danswer.search.models import SearchRequest -from danswer.search.postprocessing.postprocessing import cleanup_chunks -from danswer.search.postprocessing.postprocessing import search_postprocessing -from danswer.search.preprocessing.preprocessing import retrieval_preprocessing -from danswer.search.retrieval.search_runner import retrieve_chunks -from danswer.search.utils import inference_section_from_chunks -from danswer.search.utils import relevant_sections_to_indices from danswer.secondary_llm_flows.agentic_evaluation import evaluate_inference_section from danswer.utils.logger import setup_logger from danswer.utils.threadpool_concurrency import FunctionCall diff --git a/backend/danswer/search/postprocessing/postprocessing.py b/backend/danswer/context/search/postprocessing/postprocessing.py similarity index 96% rename from backend/danswer/search/postprocessing/postprocessing.py rename to backend/danswer/context/search/postprocessing/postprocessing.py index b4a1e48bd39..a32abca9c43 100644 --- a/backend/danswer/search/postprocessing/postprocessing.py +++ b/backend/danswer/context/search/postprocessing/postprocessing.py @@ -9,19 +9,19 @@ from danswer.configs.constants import RETURN_SEPARATOR from danswer.configs.model_configs import CROSS_ENCODER_RANGE_MAX from danswer.configs.model_configs import CROSS_ENCODER_RANGE_MIN +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.models import ChunkMetric +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceChunkUncleaned +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import MAX_METRICS_CONTENT +from danswer.context.search.models import RerankMetricsContainer +from danswer.context.search.models import SearchQuery from danswer.document_index.document_index_utils import ( translate_boost_count_to_multiplier, ) from danswer.llm.interfaces import LLM from danswer.natural_language_processing.search_nlp_models import RerankingModel -from danswer.search.enums import LLMEvaluationType -from danswer.search.models import ChunkMetric -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceChunkUncleaned -from danswer.search.models import InferenceSection -from danswer.search.models import MAX_METRICS_CONTENT -from danswer.search.models import RerankMetricsContainer -from danswer.search.models import SearchQuery from danswer.secondary_llm_flows.chunk_usefulness import llm_batch_eval_sections from danswer.utils.logger import setup_logger from danswer.utils.threadpool_concurrency import FunctionCall diff --git a/backend/danswer/search/preprocessing/access_filters.py b/backend/danswer/context/search/preprocessing/access_filters.py similarity index 91% rename from backend/danswer/search/preprocessing/access_filters.py rename to backend/danswer/context/search/preprocessing/access_filters.py index e8141864d11..d4e0c6033bf 100644 --- a/backend/danswer/search/preprocessing/access_filters.py +++ b/backend/danswer/context/search/preprocessing/access_filters.py @@ -1,8 +1,8 @@ from sqlalchemy.orm import Session from danswer.access.access import get_acl_for_user +from danswer.context.search.models import IndexFilters from danswer.db.models import User -from danswer.search.models import IndexFilters def build_access_filters_for_user(user: User | None, session: Session) -> list[str]: diff --git a/backend/danswer/search/preprocessing/preprocessing.py b/backend/danswer/context/search/preprocessing/preprocessing.py similarity index 93% rename from backend/danswer/search/preprocessing/preprocessing.py rename to backend/danswer/context/search/preprocessing/preprocessing.py index 77098dac053..fe886ccd9e0 100644 --- a/backend/danswer/search/preprocessing/preprocessing.py +++ b/backend/danswer/context/search/preprocessing/preprocessing.py @@ -9,21 +9,25 @@ from danswer.configs.chat_configs import HYBRID_ALPHA_KEYWORD from danswer.configs.chat_configs import NUM_POSTPROCESSED_RESULTS from danswer.configs.chat_configs import NUM_RETURNED_HITS +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import RecencyBiasSetting +from danswer.context.search.enums import SearchType +from danswer.context.search.models import BaseFilters +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import RerankingDetails +from danswer.context.search.models import SearchQuery +from danswer.context.search.models import SearchRequest +from danswer.context.search.preprocessing.access_filters import ( + build_access_filters_for_user, +) +from danswer.context.search.retrieval.search_runner import ( + remove_stop_words_and_punctuation, +) from danswer.db.engine import CURRENT_TENANT_ID_CONTEXTVAR from danswer.db.models import User from danswer.db.search_settings import get_current_search_settings from danswer.llm.interfaces import LLM from danswer.natural_language_processing.search_nlp_models import QueryAnalysisModel -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import RecencyBiasSetting -from danswer.search.enums import SearchType -from danswer.search.models import BaseFilters -from danswer.search.models import IndexFilters -from danswer.search.models import RerankingDetails -from danswer.search.models import SearchQuery -from danswer.search.models import SearchRequest -from danswer.search.preprocessing.access_filters import build_access_filters_for_user -from danswer.search.retrieval.search_runner import remove_stop_words_and_punctuation from danswer.secondary_llm_flows.source_filter import extract_source_filter from danswer.secondary_llm_flows.time_filter import extract_time_filter from danswer.utils.logger import setup_logger diff --git a/backend/danswer/search/retrieval/search_runner.py b/backend/danswer/context/search/retrieval/search_runner.py similarity index 95% rename from backend/danswer/search/retrieval/search_runner.py rename to backend/danswer/context/search/retrieval/search_runner.py index 30347464ff8..770a19900f2 100644 --- a/backend/danswer/search/retrieval/search_runner.py +++ b/backend/danswer/context/search/retrieval/search_runner.py @@ -6,6 +6,16 @@ from nltk.tokenize import word_tokenize # type:ignore from sqlalchemy.orm import Session +from danswer.context.search.models import ChunkMetric +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceChunkUncleaned +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import MAX_METRICS_CONTENT +from danswer.context.search.models import RetrievalMetricsContainer +from danswer.context.search.models import SearchQuery +from danswer.context.search.postprocessing.postprocessing import cleanup_chunks +from danswer.context.search.utils import inference_section_from_chunks from danswer.db.search_settings import get_current_search_settings from danswer.db.search_settings import get_multilingual_expansion from danswer.document_index.interfaces import DocumentIndex @@ -14,16 +24,6 @@ replace_invalid_doc_id_characters, ) from danswer.natural_language_processing.search_nlp_models import EmbeddingModel -from danswer.search.models import ChunkMetric -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceChunkUncleaned -from danswer.search.models import InferenceSection -from danswer.search.models import MAX_METRICS_CONTENT -from danswer.search.models import RetrievalMetricsContainer -from danswer.search.models import SearchQuery -from danswer.search.postprocessing.postprocessing import cleanup_chunks -from danswer.search.utils import inference_section_from_chunks from danswer.secondary_llm_flows.query_expansion import multilingual_query_expansion from danswer.utils.logger import setup_logger from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel diff --git a/backend/danswer/search/search_settings.py b/backend/danswer/context/search/search_settings.py similarity index 95% rename from backend/danswer/search/search_settings.py rename to backend/danswer/context/search/search_settings.py index f5870de83f1..917b2c9f532 100644 --- a/backend/danswer/search/search_settings.py +++ b/backend/danswer/context/search/search_settings.py @@ -1,9 +1,9 @@ from typing import cast from danswer.configs.constants import KV_SEARCH_SETTINGS +from danswer.context.search.models import SavedSearchSettings from danswer.key_value_store.factory import get_kv_store from danswer.key_value_store.interface import KvKeyNotFoundError -from danswer.search.models import SavedSearchSettings from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/search/utils.py b/backend/danswer/context/search/utils.py similarity index 92% rename from backend/danswer/search/utils.py rename to backend/danswer/context/search/utils.py index 21a95320ef5..ecbdaf35ffb 100644 --- a/backend/danswer/search/utils.py +++ b/backend/danswer/context/search/utils.py @@ -2,12 +2,12 @@ from typing import TypeVar from danswer.chat.models import SectionRelevancePiece +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import SavedSearchDoc +from danswer.context.search.models import SavedSearchDocWithContent +from danswer.context.search.models import SearchDoc from danswer.db.models import SearchDoc as DBSearchDoc -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceSection -from danswer.search.models import SavedSearchDoc -from danswer.search.models import SavedSearchDocWithContent -from danswer.search.models import SearchDoc T = TypeVar( diff --git a/backend/danswer/danswerbot/slack/blocks.py b/backend/danswer/danswerbot/slack/blocks.py index 4107a381554..1f689157452 100644 --- a/backend/danswer/danswerbot/slack/blocks.py +++ b/backend/danswer/danswerbot/slack/blocks.py @@ -21,6 +21,7 @@ from danswer.configs.constants import DocumentSource from danswer.configs.constants import SearchFeedbackType from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_DOCS_TO_DISPLAY +from danswer.context.search.models import SavedSearchDoc from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID from danswer.danswerbot.slack.constants import FOLLOWUP_BUTTON_ACTION_ID @@ -31,7 +32,6 @@ from danswer.danswerbot.slack.utils import build_feedback_id from danswer.danswerbot.slack.utils import remove_slack_text_interactions from danswer.danswerbot.slack.utils import translate_vespa_highlight_to_slack -from danswer.search.models import SavedSearchDoc from danswer.utils.text_processing import decode_escapes from danswer.utils.text_processing import replace_whitespaces_w_space diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py index 9026638adc4..3d5f013dca8 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py @@ -21,6 +21,10 @@ from danswer.configs.danswerbot_configs import DANSWER_FOLLOWUP_EMOJI from danswer.configs.danswerbot_configs import DANSWER_REACT_EMOJI from danswer.configs.danswerbot_configs import ENABLE_DANSWERBOT_REFLEXION +from danswer.context.search.enums import OptionalSearchSetting +from danswer.context.search.models import BaseFilters +from danswer.context.search.models import RerankingDetails +from danswer.context.search.models import RetrievalDetails from danswer.danswerbot.slack.blocks import build_documents_blocks from danswer.danswerbot.slack.blocks import build_follow_up_block from danswer.danswerbot.slack.blocks import build_qa_response_blocks @@ -48,10 +52,6 @@ from danswer.one_shot_answer.answer_question import get_search_answer from danswer.one_shot_answer.models import DirectQARequest from danswer.one_shot_answer.models import OneShotQAResponse -from danswer.search.enums import OptionalSearchSetting -from danswer.search.models import BaseFilters -from danswer.search.models import RerankingDetails -from danswer.search.models import RetrievalDetails from danswer.utils.logger import DanswerLoggingAdapter diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py index 465ce56c575..5f6cabb3406 100644 --- a/backend/danswer/danswerbot/slack/listener.py +++ b/backend/danswer/danswerbot/slack/listener.py @@ -27,6 +27,7 @@ from danswer.configs.danswerbot_configs import DANSWER_BOT_RESPOND_EVERY_CHANNEL from danswer.configs.danswerbot_configs import NOTIFY_SLACKBOT_NO_ANSWER from danswer.connectors.slack.utils import expert_info_from_slack_id +from danswer.context.search.retrieval.search_runner import download_nltk_data from danswer.danswerbot.slack.config import get_slack_channel_config_for_bot_and_channel from danswer.danswerbot.slack.config import MAX_TENANTS_PER_POD from danswer.danswerbot.slack.config import TENANT_ACQUISITION_INTERVAL @@ -75,7 +76,6 @@ from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder from danswer.one_shot_answer.models import ThreadMessage from danswer.redis.redis_pool import get_redis_client -from danswer.search.retrieval.search_runner import download_nltk_data from danswer.server.manage.models import SlackBotTokens from danswer.utils.logger import setup_logger from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable diff --git a/backend/danswer/db/chat.py b/backend/danswer/db/chat.py index 4aaee092972..a76fcccdd8d 100644 --- a/backend/danswer/db/chat.py +++ b/backend/danswer/db/chat.py @@ -18,6 +18,9 @@ from danswer.chat.models import DocumentRelevance from danswer.configs.chat_configs import HARD_DELETE_CHATS from danswer.configs.constants import MessageType +from danswer.context.search.models import RetrievalDocs +from danswer.context.search.models import SavedSearchDoc +from danswer.context.search.models import SearchDoc as ServerSearchDoc from danswer.db.models import ChatMessage from danswer.db.models import ChatMessage__SearchDoc from danswer.db.models import ChatSession @@ -31,9 +34,6 @@ from danswer.file_store.models import FileDescriptor from danswer.llm.override_models import LLMOverride from danswer.llm.override_models import PromptOverride -from danswer.search.models import RetrievalDocs -from danswer.search.models import SavedSearchDoc -from danswer.search.models import SearchDoc as ServerSearchDoc from danswer.server.query_and_chat.models import ChatMessageDetail from danswer.tools.tool_runner import ToolCallFinalResult from danswer.utils.logger import setup_logger diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index 1513fc5fc81..76e70c2d2d9 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -57,7 +57,7 @@ from danswer.file_store.models import FileDescriptor from danswer.llm.override_models import LLMOverride from danswer.llm.override_models import PromptOverride -from danswer.search.enums import RecencyBiasSetting +from danswer.context.search.enums import RecencyBiasSetting from danswer.utils.encryption import decrypt_bytes_to_string from danswer.utils.encryption import encrypt_string_to_bytes from danswer.utils.headers import HeaderItemDict diff --git a/backend/danswer/db/persona.py b/backend/danswer/db/persona.py index 5194a06717f..328979784ca 100644 --- a/backend/danswer/db/persona.py +++ b/backend/danswer/db/persona.py @@ -20,6 +20,7 @@ from danswer.configs.chat_configs import BING_API_KEY from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.constants import SLACK_BOT_PERSONA_PREFIX from danswer.db.engine import get_sqlalchemy_engine from danswer.db.models import DocumentSet @@ -33,7 +34,6 @@ from danswer.db.models import User from danswer.db.models import User__UserGroup from danswer.db.models import UserGroup -from danswer.search.enums import RecencyBiasSetting from danswer.server.features.persona.models import CreatePersonaRequest from danswer.server.features.persona.models import PersonaSnapshot from danswer.utils.logger import setup_logger diff --git a/backend/danswer/db/search_settings.py b/backend/danswer/db/search_settings.py index 342bb70b2a6..4f437eaae53 100644 --- a/backend/danswer/db/search_settings.py +++ b/backend/danswer/db/search_settings.py @@ -12,6 +12,7 @@ from danswer.configs.model_configs import OLD_DEFAULT_DOCUMENT_ENCODER_MODEL from danswer.configs.model_configs import OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM from danswer.configs.model_configs import OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS +from danswer.context.search.models import SavedSearchSettings from danswer.db.engine import get_session_with_default_tenant from danswer.db.llm import fetch_embedding_provider from danswer.db.models import CloudEmbeddingProvider @@ -21,7 +22,6 @@ from danswer.indexing.models import IndexingSetting from danswer.natural_language_processing.search_nlp_models import clean_model_name from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder -from danswer.search.models import SavedSearchSettings from danswer.server.manage.embedding.models import ( CloudEmbeddingProvider as ServerCloudEmbeddingProvider, ) diff --git a/backend/danswer/db/slack_channel_config.py b/backend/danswer/db/slack_channel_config.py index b0d0b25eca9..00e5965120a 100644 --- a/backend/danswer/db/slack_channel_config.py +++ b/backend/danswer/db/slack_channel_config.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from danswer.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.constants import SLACK_BOT_PERSONA_PREFIX from danswer.db.models import ChannelConfig from danswer.db.models import Persona @@ -15,7 +16,6 @@ from danswer.db.persona import get_default_prompt from danswer.db.persona import mark_persona_as_deleted from danswer.db.persona import upsert_persona -from danswer.search.enums import RecencyBiasSetting from danswer.utils.errors import EERequiredError from danswer.utils.variable_functionality import ( fetch_versioned_implementation_with_fallback, diff --git a/backend/danswer/document_index/document_index_utils.py b/backend/danswer/document_index/document_index_utils.py index fab7b85ef48..c6f48d1b40d 100644 --- a/backend/danswer/document_index/document_index_utils.py +++ b/backend/danswer/document_index/document_index_utils.py @@ -3,10 +3,10 @@ from sqlalchemy.orm import Session +from danswer.context.search.models import InferenceChunk from danswer.db.search_settings import get_current_search_settings from danswer.db.search_settings import get_secondary_search_settings from danswer.indexing.models import IndexChunk -from danswer.search.models import InferenceChunk DEFAULT_BATCH_SIZE = 30 diff --git a/backend/danswer/document_index/interfaces.py b/backend/danswer/document_index/interfaces.py index 07c1b24ab2e..84dcbf48473 100644 --- a/backend/danswer/document_index/interfaces.py +++ b/backend/danswer/document_index/interfaces.py @@ -4,9 +4,9 @@ from typing import Any from danswer.access.models import DocumentAccess +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceChunkUncleaned from danswer.indexing.models import DocMetadataAwareIndexChunk -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceChunkUncleaned from shared_configs.model_server_models import Embedding diff --git a/backend/danswer/document_index/vespa/chunk_retrieval.py b/backend/danswer/document_index/vespa/chunk_retrieval.py index ce95ff6fecb..6715dc76ae1 100644 --- a/backend/danswer/document_index/vespa/chunk_retrieval.py +++ b/backend/danswer/document_index/vespa/chunk_retrieval.py @@ -11,6 +11,8 @@ from retry import retry from danswer.configs.app_configs import LOG_VESPA_TIMING_INFORMATION +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceChunkUncleaned from danswer.document_index.interfaces import VespaChunkRequest from danswer.document_index.vespa.shared_utils.utils import get_vespa_http_client from danswer.document_index.vespa.shared_utils.vespa_request_builders import ( @@ -44,8 +46,6 @@ from danswer.document_index.vespa_constants import SOURCE_TYPE from danswer.document_index.vespa_constants import TITLE from danswer.document_index.vespa_constants import YQL_BASE -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceChunkUncleaned from danswer.utils.logger import setup_logger from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py index f2b9c06f916..ebe6daca1a2 100644 --- a/backend/danswer/document_index/vespa/index.py +++ b/backend/danswer/document_index/vespa/index.py @@ -22,6 +22,8 @@ from danswer.configs.chat_configs import TITLE_CONTENT_RATIO from danswer.configs.chat_configs import VESPA_SEARCHER_THREADS from danswer.configs.constants import KV_REINDEX_KEY +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceChunkUncleaned from danswer.document_index.interfaces import DocumentIndex from danswer.document_index.interfaces import DocumentInsertionRecord from danswer.document_index.interfaces import UpdateRequest @@ -68,8 +70,6 @@ from danswer.document_index.vespa_constants import YQL_BASE from danswer.indexing.models import DocMetadataAwareIndexChunk from danswer.key_value_store.factory import get_kv_store -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceChunkUncleaned from danswer.utils.batching import batch_generator from danswer.utils.logger import setup_logger from shared_configs.configs import MULTI_TENANT diff --git a/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py b/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py index e7b778f1c84..a3261288dda 100644 --- a/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py +++ b/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py @@ -3,6 +3,7 @@ from datetime import timezone from danswer.configs.constants import INDEX_SEPARATOR +from danswer.context.search.models import IndexFilters from danswer.document_index.interfaces import VespaChunkRequest from danswer.document_index.vespa_constants import ACCESS_CONTROL_LIST from danswer.document_index.vespa_constants import CHUNK_ID @@ -13,7 +14,6 @@ from danswer.document_index.vespa_constants import METADATA_LIST from danswer.document_index.vespa_constants import SOURCE_TYPE from danswer.document_index.vespa_constants import TENANT_ID -from danswer.search.models import IndexFilters from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/llm/answering/prompts/citations_prompt.py b/backend/danswer/llm/answering/prompts/citations_prompt.py index b7ca7797e88..1ff48432b86 100644 --- a/backend/danswer/llm/answering/prompts/citations_prompt.py +++ b/backend/danswer/llm/answering/prompts/citations_prompt.py @@ -3,6 +3,7 @@ from danswer.chat.models import LlmDoc from danswer.configs.model_configs import GEN_AI_SINGLE_USER_MESSAGE_EXPECTED_MAX_TOKENS +from danswer.context.search.models import InferenceChunk from danswer.db.models import Persona from danswer.db.persona import get_default_prompt__read_only from danswer.db.search_settings import get_multilingual_expansion @@ -29,7 +30,6 @@ from danswer.prompts.token_counts import CITATION_REMINDER_TOKEN_CNT from danswer.prompts.token_counts import CITATION_STATEMENT_TOKEN_CNT from danswer.prompts.token_counts import LANGUAGE_HINT_TOKEN_CNT -from danswer.search.models import InferenceChunk from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/llm/answering/prompts/quotes_prompt.py b/backend/danswer/llm/answering/prompts/quotes_prompt.py index 42f736b627d..3cdaaefcfdb 100644 --- a/backend/danswer/llm/answering/prompts/quotes_prompt.py +++ b/backend/danswer/llm/answering/prompts/quotes_prompt.py @@ -3,6 +3,7 @@ from danswer.chat.models import LlmDoc from danswer.configs.chat_configs import LANGUAGE_HINT from danswer.configs.chat_configs import QA_PROMPT_OVERRIDE +from danswer.context.search.models import InferenceChunk from danswer.db.search_settings import get_multilingual_expansion from danswer.llm.answering.models import PromptConfig from danswer.llm.utils import message_to_prompt_and_imgs @@ -12,7 +13,6 @@ from danswer.prompts.direct_qa_prompts import WEAK_LLM_PROMPT from danswer.prompts.prompt_utils import add_date_time_to_prompt from danswer.prompts.prompt_utils import build_complete_context_str -from danswer.search.models import InferenceChunk def _build_weak_llm_quotes_prompt( diff --git a/backend/danswer/llm/answering/prune_and_merge.py b/backend/danswer/llm/answering/prune_and_merge.py index 690a5d2280d..21ea2226d97 100644 --- a/backend/danswer/llm/answering/prune_and_merge.py +++ b/backend/danswer/llm/answering/prune_and_merge.py @@ -10,6 +10,8 @@ ) from danswer.configs.constants import IGNORE_FOR_QA from danswer.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceSection from danswer.llm.answering.models import ContextualPruningConfig from danswer.llm.answering.models import PromptConfig from danswer.llm.answering.prompts.citations_prompt import compute_max_document_tokens @@ -17,8 +19,6 @@ from danswer.natural_language_processing.utils import get_tokenizer from danswer.natural_language_processing.utils import tokenizer_trim_content from danswer.prompts.prompt_utils import build_doc_context_str -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceSection from danswer.tools.tool_implementations.search.search_utils import section_to_dict from danswer.utils.logger import setup_logger diff --git a/backend/danswer/llm/answering/stream_processing/quotes_processing.py b/backend/danswer/llm/answering/stream_processing/quotes_processing.py index bcd6d156467..1f1afc1aaba 100644 --- a/backend/danswer/llm/answering/stream_processing/quotes_processing.py +++ b/backend/danswer/llm/answering/stream_processing/quotes_processing.py @@ -12,9 +12,9 @@ from danswer.chat.models import DanswerQuotes from danswer.chat.models import LlmDoc from danswer.configs.chat_configs import QUOTE_ALLOWED_ERROR_PERCENT +from danswer.context.search.models import InferenceChunk from danswer.prompts.constants import ANSWER_PAT from danswer.prompts.constants import QUOTE_PAT -from danswer.search.models import InferenceChunk from danswer.utils.logger import setup_logger from danswer.utils.text_processing import clean_model_quote from danswer.utils.text_processing import clean_up_code_blocks diff --git a/backend/danswer/llm/answering/stream_processing/utils.py b/backend/danswer/llm/answering/stream_processing/utils.py index b4fb83747de..4f328fc7240 100644 --- a/backend/danswer/llm/answering/stream_processing/utils.py +++ b/backend/danswer/llm/answering/stream_processing/utils.py @@ -3,7 +3,7 @@ from pydantic import BaseModel from danswer.chat.models import LlmDoc -from danswer.search.models import InferenceChunk +from danswer.context.search.models import InferenceChunk class DocumentIdOrderMapping(BaseModel): diff --git a/backend/danswer/natural_language_processing/utils.py b/backend/danswer/natural_language_processing/utils.py index 56631495a4e..b46757728f9 100644 --- a/backend/danswer/natural_language_processing/utils.py +++ b/backend/danswer/natural_language_processing/utils.py @@ -7,7 +7,7 @@ from danswer.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE from danswer.configs.model_configs import DOCUMENT_ENCODER_MODEL -from danswer.search.models import InferenceChunk +from danswer.context.search.models import InferenceChunk from danswer.utils.logger import setup_logger from shared_configs.enums import EmbeddingProvider diff --git a/backend/danswer/one_shot_answer/answer_question.py b/backend/danswer/one_shot_answer/answer_question.py index 06b6b67e9d4..9f8ce99231b 100644 --- a/backend/danswer/one_shot_answer/answer_question.py +++ b/backend/danswer/one_shot_answer/answer_question.py @@ -18,6 +18,11 @@ from danswer.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT from danswer.configs.chat_configs import QA_TIMEOUT from danswer.configs.constants import MessageType +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.models import RerankMetricsContainer +from danswer.context.search.models import RetrievalMetricsContainer +from danswer.context.search.utils import chunks_or_sections_to_search_docs +from danswer.context.search.utils import dedupe_documents from danswer.db.chat import create_chat_session from danswer.db.chat import create_db_search_doc from danswer.db.chat import create_new_chat_message @@ -42,11 +47,6 @@ from danswer.one_shot_answer.models import OneShotQAResponse from danswer.one_shot_answer.models import QueryRephrase from danswer.one_shot_answer.qa_utils import combine_message_thread -from danswer.search.enums import LLMEvaluationType -from danswer.search.models import RerankMetricsContainer -from danswer.search.models import RetrievalMetricsContainer -from danswer.search.utils import chunks_or_sections_to_search_docs -from danswer.search.utils import dedupe_documents from danswer.secondary_llm_flows.answer_validation import get_answer_validity from danswer.secondary_llm_flows.query_expansion import thread_based_query_rephrase from danswer.server.query_and_chat.models import ChatMessageDetail diff --git a/backend/danswer/one_shot_answer/models.py b/backend/danswer/one_shot_answer/models.py index 735fc12bbb9..21463867d28 100644 --- a/backend/danswer/one_shot_answer/models.py +++ b/backend/danswer/one_shot_answer/models.py @@ -9,12 +9,12 @@ from danswer.chat.models import DanswerQuotes from danswer.chat.models import QADocsResponse from danswer.configs.constants import MessageType -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import RecencyBiasSetting -from danswer.search.enums import SearchType -from danswer.search.models import ChunkContext -from danswer.search.models import RerankingDetails -from danswer.search.models import RetrievalDetails +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import RecencyBiasSetting +from danswer.context.search.enums import SearchType +from danswer.context.search.models import ChunkContext +from danswer.context.search.models import RerankingDetails +from danswer.context.search.models import RetrievalDetails class QueryRephrase(BaseModel): diff --git a/backend/danswer/prompts/prompt_utils.py b/backend/danswer/prompts/prompt_utils.py index 5e1a3e298fb..4195926dbbe 100644 --- a/backend/danswer/prompts/prompt_utils.py +++ b/backend/danswer/prompts/prompt_utils.py @@ -7,12 +7,12 @@ from danswer.chat.models import LlmDoc from danswer.configs.chat_configs import LANGUAGE_HINT from danswer.configs.constants import DocumentSource +from danswer.context.search.models import InferenceChunk from danswer.db.models import Prompt from danswer.llm.answering.models import PromptConfig from danswer.prompts.chat_prompts import ADDITIONAL_INFO from danswer.prompts.chat_prompts import CITATION_REMINDER from danswer.prompts.constants import CODE_BLOCK_PAT -from danswer.search.models import InferenceChunk from danswer.utils.logger import setup_logger diff --git a/backend/danswer/secondary_llm_flows/agentic_evaluation.py b/backend/danswer/secondary_llm_flows/agentic_evaluation.py index 03121e3cf1d..2da61f4cb51 100644 --- a/backend/danswer/secondary_llm_flows/agentic_evaluation.py +++ b/backend/danswer/secondary_llm_flows/agentic_evaluation.py @@ -1,12 +1,12 @@ import re from danswer.chat.models import SectionRelevancePiece +from danswer.context.search.models import InferenceSection from danswer.llm.interfaces import LLM from danswer.llm.utils import dict_based_prompt_to_langchain_prompt from danswer.llm.utils import message_to_string from danswer.prompts.agentic_evaluation import AGENTIC_SEARCH_SYSTEM_PROMPT from danswer.prompts.agentic_evaluation import AGENTIC_SEARCH_USER_PROMPT -from danswer.search.models import InferenceSection from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/seeding/load_yamls.py b/backend/danswer/seeding/load_yamls.py index e8a19c158b2..0046352679c 100644 --- a/backend/danswer/seeding/load_yamls.py +++ b/backend/danswer/seeding/load_yamls.py @@ -5,6 +5,7 @@ from danswer.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT from danswer.configs.chat_configs import PERSONAS_YAML from danswer.configs.chat_configs import PROMPTS_YAML +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.document_set import get_or_create_document_set_by_name from danswer.db.input_prompt import insert_input_prompt_if_not_exists from danswer.db.models import DocumentSet as DocumentSetDBModel @@ -14,7 +15,6 @@ from danswer.db.persona import get_prompt_by_name from danswer.db.persona import upsert_persona from danswer.db.persona import upsert_prompt -from danswer.search.enums import RecencyBiasSetting def load_prompts_from_yaml( diff --git a/backend/danswer/server/documents/document.py b/backend/danswer/server/documents/document.py index bf8cdbcef44..0b8d8e744bd 100644 --- a/backend/danswer/server/documents/document.py +++ b/backend/danswer/server/documents/document.py @@ -5,6 +5,10 @@ from sqlalchemy.orm import Session from danswer.auth.users import current_user +from danswer.context.search.models import IndexFilters +from danswer.context.search.preprocessing.access_filters import ( + build_access_filters_for_user, +) from danswer.db.engine import get_session from danswer.db.models import User from danswer.db.search_settings import get_current_search_settings @@ -12,8 +16,6 @@ from danswer.document_index.interfaces import VespaChunkRequest from danswer.natural_language_processing.utils import get_tokenizer from danswer.prompts.prompt_utils import build_doc_context_str -from danswer.search.models import IndexFilters -from danswer.search.preprocessing.access_filters import build_access_filters_for_user from danswer.server.documents.models import ChunkInfo from danswer.server.documents.models import DocumentInfo diff --git a/backend/danswer/server/features/persona/models.py b/backend/danswer/server/features/persona/models.py index 4331d0640da..f32a62090d2 100644 --- a/backend/danswer/server/features/persona/models.py +++ b/backend/danswer/server/features/persona/models.py @@ -4,10 +4,10 @@ from pydantic import BaseModel from pydantic import Field +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.models import Persona from danswer.db.models import PersonaCategory from danswer.db.models import StarterMessage -from danswer.search.enums import RecencyBiasSetting from danswer.server.features.document_set.models import DocumentSet from danswer.server.features.prompt.models import PromptSnapshot from danswer.server.features.tool.models import ToolSnapshot diff --git a/backend/danswer/server/gpts/api.py b/backend/danswer/server/gpts/api.py index 1bebc3bfc1e..1d0684cf61a 100644 --- a/backend/danswer/server/gpts/api.py +++ b/backend/danswer/server/gpts/api.py @@ -6,11 +6,11 @@ from pydantic import BaseModel from sqlalchemy.orm import Session +from danswer.context.search.models import SearchRequest +from danswer.context.search.pipeline import SearchPipeline from danswer.db.engine import get_session from danswer.db.models import User from danswer.llm.factory import get_default_llms -from danswer.search.models import SearchRequest -from danswer.search.pipeline import SearchPipeline from danswer.server.danswer_api.ingestion import api_key_dep from danswer.utils.logger import setup_logger diff --git a/backend/danswer/server/manage/models.py b/backend/danswer/server/manage/models.py index 4ece3c7f169..74a3a774e21 100644 --- a/backend/danswer/server/manage/models.py +++ b/backend/danswer/server/manage/models.py @@ -10,6 +10,7 @@ from danswer.auth.schemas import UserRole from danswer.configs.app_configs import TRACK_EXTERNAL_IDP_EXPIRY from danswer.configs.constants import AuthType +from danswer.context.search.models import SavedSearchSettings from danswer.danswerbot.slack.config import VALID_SLACK_FILTERS from danswer.db.models import AllowedAnswerFilters from danswer.db.models import ChannelConfig @@ -17,7 +18,6 @@ from danswer.db.models import SlackBotResponseType from danswer.db.models import SlackChannelConfig as SlackChannelConfigModel from danswer.db.models import User -from danswer.search.models import SavedSearchSettings from danswer.server.features.persona.models import PersonaSnapshot from danswer.server.models import FullUserSnapshot from danswer.server.models import InvitedUserSnapshot diff --git a/backend/danswer/server/manage/search_settings.py b/backend/danswer/server/manage/search_settings.py index 79f690e5db6..5b8d7d8e1bb 100644 --- a/backend/danswer/server/manage/search_settings.py +++ b/backend/danswer/server/manage/search_settings.py @@ -7,6 +7,8 @@ from danswer.auth.users import current_admin_user from danswer.auth.users import current_user from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP +from danswer.context.search.models import SavedSearchSettings +from danswer.context.search.models import SearchSettingsCreationRequest from danswer.db.connector_credential_pair import get_connector_credential_pairs from danswer.db.connector_credential_pair import resync_cc_pair from danswer.db.engine import get_session @@ -25,8 +27,6 @@ from danswer.file_processing.unstructured import get_unstructured_api_key from danswer.file_processing.unstructured import update_unstructured_api_key from danswer.natural_language_processing.search_nlp_models import clean_model_name -from danswer.search.models import SavedSearchSettings -from danswer.search.models import SearchSettingsCreationRequest from danswer.server.manage.embedding.models import SearchSettingsDeleteRequest from danswer.server.manage.models import FullModelVersionResponse from danswer.server.models import IdReturn diff --git a/backend/danswer/server/openai_assistants_api/asssistants_api.py b/backend/danswer/server/openai_assistants_api/asssistants_api.py index 66a04e8969c..000944213da 100644 --- a/backend/danswer/server/openai_assistants_api/asssistants_api.py +++ b/backend/danswer/server/openai_assistants_api/asssistants_api.py @@ -10,6 +10,7 @@ from sqlalchemy.orm import Session from danswer.auth.users import current_user +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.engine import get_session from danswer.db.models import Persona from danswer.db.models import User @@ -19,7 +20,6 @@ from danswer.db.persona import upsert_persona from danswer.db.persona import upsert_prompt from danswer.db.tools import get_tool_by_name -from danswer.search.enums import RecencyBiasSetting from danswer.utils.logger import setup_logger logger = setup_logger() diff --git a/backend/danswer/server/openai_assistants_api/runs_api.py b/backend/danswer/server/openai_assistants_api/runs_api.py index 616afcad168..44bfaa3aca4 100644 --- a/backend/danswer/server/openai_assistants_api/runs_api.py +++ b/backend/danswer/server/openai_assistants_api/runs_api.py @@ -12,6 +12,7 @@ from danswer.auth.users import current_user from danswer.chat.process_message import stream_chat_message_objects from danswer.configs.constants import MessageType +from danswer.context.search.models import RetrievalDetails from danswer.db.chat import create_new_chat_message from danswer.db.chat import get_chat_message from danswer.db.chat import get_chat_messages_by_session @@ -20,7 +21,6 @@ from danswer.db.engine import get_session from danswer.db.models import ChatMessage from danswer.db.models import User -from danswer.search.models import RetrievalDetails from danswer.server.query_and_chat.models import ChatMessageDetail from danswer.server.query_and_chat.models import CreateChatMessageRequest from danswer.tools.tool_implementations.search.search_tool import SearchTool diff --git a/backend/danswer/server/query_and_chat/models.py b/backend/danswer/server/query_and_chat/models.py index 6e905a20708..c316435996e 100644 --- a/backend/danswer/server/query_and_chat/models.py +++ b/backend/danswer/server/query_and_chat/models.py @@ -9,15 +9,15 @@ from danswer.configs.constants import DocumentSource from danswer.configs.constants import MessageType from danswer.configs.constants import SearchFeedbackType +from danswer.context.search.models import BaseFilters +from danswer.context.search.models import ChunkContext +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.models import SearchDoc +from danswer.context.search.models import Tag from danswer.db.enums import ChatSessionSharedStatus from danswer.file_store.models import FileDescriptor from danswer.llm.override_models import LLMOverride from danswer.llm.override_models import PromptOverride -from danswer.search.models import BaseFilters -from danswer.search.models import ChunkContext -from danswer.search.models import RetrievalDetails -from danswer.search.models import SearchDoc -from danswer.search.models import Tag from danswer.tools.models import ToolCallFinalResult diff --git a/backend/danswer/server/query_and_chat/query_backend.py b/backend/danswer/server/query_and_chat/query_backend.py index 3636de5080c..6fb848dfa38 100644 --- a/backend/danswer/server/query_and_chat/query_backend.py +++ b/backend/danswer/server/query_and_chat/query_backend.py @@ -13,6 +13,12 @@ from danswer.auth.users import current_user from danswer.configs.constants import DocumentSource from danswer.configs.constants import MessageType +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import SearchDoc +from danswer.context.search.preprocessing.access_filters import ( + build_access_filters_for_user, +) +from danswer.context.search.utils import chunks_or_sections_to_search_docs from danswer.db.chat import get_chat_messages_by_session from danswer.db.chat import get_chat_session_by_id from danswer.db.chat import get_chat_sessions_by_user @@ -28,10 +34,6 @@ from danswer.document_index.vespa.index import VespaIndex from danswer.one_shot_answer.answer_question import stream_search_answer from danswer.one_shot_answer.models import DirectQARequest -from danswer.search.models import IndexFilters -from danswer.search.models import SearchDoc -from danswer.search.preprocessing.access_filters import build_access_filters_for_user -from danswer.search.utils import chunks_or_sections_to_search_docs from danswer.secondary_llm_flows.query_validation import get_query_answerability from danswer.secondary_llm_flows.query_validation import stream_query_answerability from danswer.server.query_and_chat.models import AdminSearchRequest diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py index 8b2e6d209b3..99173821a45 100644 --- a/backend/danswer/setup.py +++ b/backend/danswer/setup.py @@ -9,6 +9,8 @@ from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION from danswer.configs.model_configs import GEN_AI_API_KEY from danswer.configs.model_configs import GEN_AI_MODEL_VERSION +from danswer.context.search.models import SavedSearchSettings +from danswer.context.search.retrieval.search_runner import download_nltk_data from danswer.db.connector import check_connectors_exist from danswer.db.connector import create_initial_default_connector from danswer.db.connector_credential_pair import associate_default_cc_pair @@ -36,8 +38,6 @@ from danswer.natural_language_processing.search_nlp_models import EmbeddingModel from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder -from danswer.search.models import SavedSearchSettings -from danswer.search.retrieval.search_runner import download_nltk_data from danswer.seeding.load_docs import seed_initial_documents from danswer.seeding.load_yamls import load_chat_yamls from danswer.server.manage.llm.models import LLMProviderUpsertRequest diff --git a/backend/danswer/tools/tool_constructor.py b/backend/danswer/tools/tool_constructor.py index dacbe5ad112..a8fb5706dc2 100644 --- a/backend/danswer/tools/tool_constructor.py +++ b/backend/danswer/tools/tool_constructor.py @@ -11,6 +11,9 @@ from danswer.configs.app_configs import AZURE_DALLE_DEPLOYMENT_NAME from danswer.configs.chat_configs import BING_API_KEY from danswer.configs.model_configs import GEN_AI_TEMPERATURE +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import RetrievalDetails from danswer.db.llm import fetch_existing_llm_providers from danswer.db.models import Persona from danswer.db.models import User @@ -22,9 +25,6 @@ from danswer.llm.interfaces import LLM from danswer.llm.interfaces import LLMConfig from danswer.natural_language_processing.utils import get_tokenizer -from danswer.search.enums import LLMEvaluationType -from danswer.search.models import InferenceSection -from danswer.search.models import RetrievalDetails from danswer.tools.built_in_tools import get_built_in_tool_by_id from danswer.tools.models import DynamicSchemaInfo from danswer.tools.tool import Tool diff --git a/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py b/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py index 4ec2dd5cb8b..fd59b08abe1 100644 --- a/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py +++ b/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py @@ -10,6 +10,7 @@ from danswer.chat.models import LlmDoc from danswer.configs.constants import DocumentSource from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF +from danswer.context.search.models import SearchDoc from danswer.llm.answering.models import AnswerStyleConfig from danswer.llm.answering.models import PreviousMessage from danswer.llm.answering.models import PromptConfig @@ -18,7 +19,6 @@ from danswer.llm.utils import message_to_string from danswer.prompts.chat_prompts import INTERNET_SEARCH_QUERY_REPHRASE from danswer.prompts.constants import GENERAL_SEP_PAT -from danswer.search.models import SearchDoc from danswer.secondary_llm_flows.query_expansion import history_based_query_rephrase from danswer.tools.message import ToolCallSummary from danswer.tools.models import ToolResponse diff --git a/backend/danswer/tools/tool_implementations/search/search_tool.py b/backend/danswer/tools/tool_implementations/search/search_tool.py index e92bc34dc57..0a7be7e3885 100644 --- a/backend/danswer/tools/tool_implementations/search/search_tool.py +++ b/backend/danswer/tools/tool_implementations/search/search_tool.py @@ -14,6 +14,14 @@ from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW from danswer.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import QueryFlow +from danswer.context.search.enums import SearchType +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import InferenceSection +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.models import SearchRequest +from danswer.context.search.pipeline import SearchPipeline from danswer.db.models import Persona from danswer.db.models import User from danswer.llm.answering.llm_response_handler import LLMCall @@ -27,14 +35,6 @@ from danswer.llm.answering.prune_and_merge import prune_and_merge_sections from danswer.llm.answering.prune_and_merge import prune_sections from danswer.llm.interfaces import LLM -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import QueryFlow -from danswer.search.enums import SearchType -from danswer.search.models import IndexFilters -from danswer.search.models import InferenceSection -from danswer.search.models import RetrievalDetails -from danswer.search.models import SearchRequest -from danswer.search.pipeline import SearchPipeline from danswer.secondary_llm_flows.choose_search import check_if_need_search from danswer.secondary_llm_flows.query_expansion import history_based_query_rephrase from danswer.tools.message import ToolCallSummary diff --git a/backend/danswer/tools/tool_implementations/search/search_utils.py b/backend/danswer/tools/tool_implementations/search/search_utils.py index 5076632a694..5eef0b891f0 100644 --- a/backend/danswer/tools/tool_implementations/search/search_utils.py +++ b/backend/danswer/tools/tool_implementations/search/search_utils.py @@ -1,6 +1,6 @@ from danswer.chat.models import LlmDoc +from danswer.context.search.models import InferenceSection from danswer.prompts.prompt_utils import clean_up_source -from danswer.search.models import InferenceSection def llm_doc_to_dict(llm_doc: LlmDoc, doc_num: int) -> dict: diff --git a/backend/ee/danswer/server/query_and_chat/chat_backend.py b/backend/ee/danswer/server/query_and_chat/chat_backend.py index b25ed8357d9..ef707cbfb24 100644 --- a/backend/ee/danswer/server/query_and_chat/chat_backend.py +++ b/backend/ee/danswer/server/query_and_chat/chat_backend.py @@ -18,6 +18,9 @@ from danswer.chat.process_message import stream_chat_message_objects from danswer.configs.constants import MessageType from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTAGE +from danswer.context.search.models import OptionalSearchSetting +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.models import SavedSearchDoc from danswer.db.chat import create_chat_session from danswer.db.chat import create_new_chat_message from danswer.db.chat import get_or_create_root_message @@ -27,9 +30,6 @@ from danswer.llm.utils import get_max_input_tokens from danswer.natural_language_processing.utils import get_tokenizer from danswer.one_shot_answer.qa_utils import combine_message_thread -from danswer.search.models import OptionalSearchSetting -from danswer.search.models import RetrievalDetails -from danswer.search.models import SavedSearchDoc from danswer.secondary_llm_flows.query_expansion import thread_based_query_rephrase from danswer.server.query_and_chat.models import ChatMessageDetail from danswer.server.query_and_chat.models import CreateChatMessageRequest diff --git a/backend/ee/danswer/server/query_and_chat/models.py b/backend/ee/danswer/server/query_and_chat/models.py index 4baf17ac8ce..1fd37a21145 100644 --- a/backend/ee/danswer/server/query_and_chat/models.py +++ b/backend/ee/danswer/server/query_and_chat/models.py @@ -4,13 +4,13 @@ from pydantic import Field from danswer.configs.constants import DocumentSource +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import SearchType +from danswer.context.search.models import ChunkContext +from danswer.context.search.models import RerankingDetails +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.models import SavedSearchDoc from danswer.one_shot_answer.models import ThreadMessage -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import SearchType -from danswer.search.models import ChunkContext -from danswer.search.models import RerankingDetails -from danswer.search.models import RetrievalDetails -from danswer.search.models import SavedSearchDoc from ee.danswer.server.manage.models import StandardAnswer diff --git a/backend/ee/danswer/server/query_and_chat/query_backend.py b/backend/ee/danswer/server/query_and_chat/query_backend.py index 59e61ba12df..0b380d5d3f7 100644 --- a/backend/ee/danswer/server/query_and_chat/query_backend.py +++ b/backend/ee/danswer/server/query_and_chat/query_backend.py @@ -6,6 +6,12 @@ from danswer.auth.users import current_user from danswer.configs.danswerbot_configs import DANSWER_BOT_TARGET_CHUNK_PERCENTAGE +from danswer.context.search.models import SavedSearchDocWithContent +from danswer.context.search.models import SearchRequest +from danswer.context.search.pipeline import SearchPipeline +from danswer.context.search.utils import dedupe_documents +from danswer.context.search.utils import drop_llm_indices +from danswer.context.search.utils import relevant_sections_to_indices from danswer.db.engine import get_session from danswer.db.models import User from danswer.db.persona import get_persona_by_id @@ -19,12 +25,6 @@ from danswer.one_shot_answer.answer_question import get_search_answer from danswer.one_shot_answer.models import DirectQARequest from danswer.one_shot_answer.models import OneShotQAResponse -from danswer.search.models import SavedSearchDocWithContent -from danswer.search.models import SearchRequest -from danswer.search.pipeline import SearchPipeline -from danswer.search.utils import dedupe_documents -from danswer.search.utils import drop_llm_indices -from danswer.search.utils import relevant_sections_to_indices from danswer.utils.logger import setup_logger from ee.danswer.danswerbot.slack.handlers.handle_standard_answers import ( oneoff_standard_answers, diff --git a/backend/ee/danswer/server/seeding.py b/backend/ee/danswer/server/seeding.py index feb10cc19ce..7aa87379221 100644 --- a/backend/ee/danswer/server/seeding.py +++ b/backend/ee/danswer/server/seeding.py @@ -7,12 +7,12 @@ from pydantic import BaseModel from sqlalchemy.orm import Session +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.engine import get_session_context_manager from danswer.db.llm import update_default_provider from danswer.db.llm import upsert_llm_provider from danswer.db.models import Tool from danswer.db.persona import upsert_persona -from danswer.search.enums import RecencyBiasSetting from danswer.server.features.persona.models import CreatePersonaRequest from danswer.server.manage.llm.models import LLMProviderUpsertRequest from danswer.server.settings.models import Settings diff --git a/backend/scripts/query_time_check/test_query_times.py b/backend/scripts/query_time_check/test_query_times.py index c839fc610e1..144b2354f93 100644 --- a/backend/scripts/query_time_check/test_query_times.py +++ b/backend/scripts/query_time_check/test_query_times.py @@ -6,10 +6,10 @@ from danswer.configs.constants import DocumentSource from danswer.configs.model_configs import DOC_EMBEDDING_DIM +from danswer.context.search.models import IndexFilters from danswer.db.engine import get_session_context_manager from danswer.db.search_settings import get_current_search_settings from danswer.document_index.vespa.index import VespaIndex -from danswer.search.models import IndexFilters from scripts.query_time_check.seed_dummy_docs import TOTAL_ACL_ENTRIES_PER_CATEGORY from scripts.query_time_check.seed_dummy_docs import TOTAL_DOC_SETS from shared_configs.model_server_models import Embedding diff --git a/backend/tests/integration/common_utils/managers/chat.py b/backend/tests/integration/common_utils/managers/chat.py index 5927d633068..106aa26a791 100644 --- a/backend/tests/integration/common_utils/managers/chat.py +++ b/backend/tests/integration/common_utils/managers/chat.py @@ -4,12 +4,12 @@ import requests from requests.models import Response +from danswer.context.search.models import RetrievalDetails from danswer.file_store.models import FileDescriptor from danswer.llm.override_models import LLMOverride from danswer.llm.override_models import PromptOverride from danswer.one_shot_answer.models import DirectQARequest from danswer.one_shot_answer.models import ThreadMessage -from danswer.search.models import RetrievalDetails from danswer.server.query_and_chat.models import ChatSessionCreationRequest from danswer.server.query_and_chat.models import CreateChatMessageRequest from tests.integration.common_utils.constants import API_SERVER_URL diff --git a/backend/tests/integration/common_utils/managers/document_search.py b/backend/tests/integration/common_utils/managers/document_search.py index 4fe2442b69a..82fdaf27db1 100644 --- a/backend/tests/integration/common_utils/managers/document_search.py +++ b/backend/tests/integration/common_utils/managers/document_search.py @@ -1,9 +1,9 @@ import requests -from danswer.search.enums import LLMEvaluationType -from danswer.search.enums import SearchType -from danswer.search.models import RetrievalDetails -from danswer.search.models import SavedSearchDocWithContent +from danswer.context.search.enums import LLMEvaluationType +from danswer.context.search.enums import SearchType +from danswer.context.search.models import RetrievalDetails +from danswer.context.search.models import SavedSearchDocWithContent from ee.danswer.server.query_and_chat.models import DocumentSearchRequest from tests.integration.common_utils.constants import API_SERVER_URL from tests.integration.common_utils.constants import GENERAL_HEADERS diff --git a/backend/tests/integration/common_utils/managers/persona.py b/backend/tests/integration/common_utils/managers/persona.py index 147e6bf0bb8..de2d9db25c1 100644 --- a/backend/tests/integration/common_utils/managers/persona.py +++ b/backend/tests/integration/common_utils/managers/persona.py @@ -2,7 +2,7 @@ import requests -from danswer.search.enums import RecencyBiasSetting +from danswer.context.search.enums import RecencyBiasSetting from danswer.server.features.persona.models import PersonaSnapshot from tests.integration.common_utils.constants import API_SERVER_URL from tests.integration.common_utils.constants import GENERAL_HEADERS diff --git a/backend/tests/integration/common_utils/test_models.py b/backend/tests/integration/common_utils/test_models.py index c0b9c92a577..65a90259d8b 100644 --- a/backend/tests/integration/common_utils/test_models.py +++ b/backend/tests/integration/common_utils/test_models.py @@ -5,8 +5,8 @@ from pydantic import Field from danswer.auth.schemas import UserRole +from danswer.context.search.enums import RecencyBiasSetting from danswer.db.enums import AccessType -from danswer.search.enums import RecencyBiasSetting from danswer.server.documents.models import DocumentSource from danswer.server.documents.models import InputType diff --git a/backend/tests/regression/answer_quality/api_utils.py b/backend/tests/regression/answer_quality/api_utils.py index 5a46032c62f..28406c061b8 100644 --- a/backend/tests/regression/answer_quality/api_utils.py +++ b/backend/tests/regression/answer_quality/api_utils.py @@ -4,12 +4,12 @@ from danswer.configs.constants import DocumentSource from danswer.configs.constants import MessageType from danswer.connectors.models import InputType +from danswer.context.search.models import IndexFilters +from danswer.context.search.models import OptionalSearchSetting +from danswer.context.search.models import RetrievalDetails from danswer.db.enums import IndexingStatus from danswer.one_shot_answer.models import DirectQARequest from danswer.one_shot_answer.models import ThreadMessage -from danswer.search.models import IndexFilters -from danswer.search.models import OptionalSearchSetting -from danswer.search.models import RetrievalDetails from danswer.server.documents.models import ConnectorBase from tests.regression.answer_quality.cli_utils import get_api_server_host_port diff --git a/backend/tests/unit/danswer/direct_qa/test_qa_utils.py b/backend/tests/unit/danswer/direct_qa/test_qa_utils.py index ad6baaa365a..bcbd76f4e12 100644 --- a/backend/tests/unit/danswer/direct_qa/test_qa_utils.py +++ b/backend/tests/unit/danswer/direct_qa/test_qa_utils.py @@ -3,13 +3,13 @@ import pytest from danswer.configs.constants import DocumentSource +from danswer.context.search.models import InferenceChunk from danswer.llm.answering.stream_processing.quotes_processing import ( match_quotes_to_docs, ) from danswer.llm.answering.stream_processing.quotes_processing import ( separate_answer_quotes, ) -from danswer.search.models import InferenceChunk def test_passed_in_quotes() -> None: diff --git a/backend/tests/unit/danswer/llm/answering/test_prune_and_merge.py b/backend/tests/unit/danswer/llm/answering/test_prune_and_merge.py index 9d28339a1f5..c71d9109007 100644 --- a/backend/tests/unit/danswer/llm/answering/test_prune_and_merge.py +++ b/backend/tests/unit/danswer/llm/answering/test_prune_and_merge.py @@ -1,9 +1,9 @@ import pytest from danswer.configs.constants import DocumentSource +from danswer.context.search.models import InferenceChunk +from danswer.context.search.models import InferenceSection from danswer.llm.answering.prune_and_merge import _merge_sections -from danswer.search.models import InferenceChunk -from danswer.search.models import InferenceSection # This large test accounts for all of the following: