From 1fe702c678b6c33323d4ff0601f0364d620da4b4 Mon Sep 17 00:00:00 2001 From: hagen-danswer Date: Fri, 22 Nov 2024 16:42:15 -0800 Subject: [PATCH] Increased slim doc batch size for confluence connector (#3221) --- backend/danswer/connectors/confluence/connector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py index c9be6676fa7..0e09a4aed61 100644 --- a/backend/danswer/connectors/confluence/connector.py +++ b/backend/danswer/connectors/confluence/connector.py @@ -51,6 +51,8 @@ "restrictions.read.restrictions.group", ] +_SLIM_DOC_BATCH_SIZE = 1000 + class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector): def __init__( @@ -263,6 +265,7 @@ def retrieve_all_slim_documents( for page in self.confluence_client.cql_paginate_all_expansions( cql=page_query, expand=restrictions_expand, + limit=_SLIM_DOC_BATCH_SIZE, ): # If the page has restrictions, add them to the perm_sync_data # These will be used by doc_sync.py to sync permissions @@ -286,6 +289,7 @@ def retrieve_all_slim_documents( for attachment in self.confluence_client.cql_paginate_all_expansions( cql=attachment_cql, expand=restrictions_expand, + limit=_SLIM_DOC_BATCH_SIZE, ): doc_metadata_list.append( SlimDocument(