Increased slim doc batch size for confluence connector (onyx-dot-app#…

…3221)
ahmadassaf · Nov 25, 2024 · 1fe702c · 1fe702c
1 parent ed90ddb
commit 1fe702c
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
@@ -51,6 +51,8 @@
     "restrictions.read.restrictions.group",
 ]
 
+_SLIM_DOC_BATCH_SIZE = 1000
+
 
 class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
@@ -263,6 +265,7 @@ def retrieve_all_slim_documents(
         for page in self.confluence_client.cql_paginate_all_expansions(
             cql=page_query,
             expand=restrictions_expand,
+            limit=_SLIM_DOC_BATCH_SIZE,
         ):
             # If the page has restrictions, add them to the perm_sync_data
             # These will be used by doc_sync.py to sync permissions
@@ -286,6 +289,7 @@ def retrieve_all_slim_documents(
             for attachment in self.confluence_client.cql_paginate_all_expansions(
                 cql=attachment_cql,
                 expand=restrictions_expand,
+                limit=_SLIM_DOC_BATCH_SIZE,
             ):
                 doc_metadata_list.append(
                     SlimDocument(