Skip to content

Commit

Permalink
now chunk length is a parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
Ilija Vukotic authored and BenGalewsky committed Jun 8, 2022
1 parent f66d0d3 commit e4bf4bf
Showing 1 changed file with 2 additions and 3 deletions.
5 changes: 2 additions & 3 deletions src/servicex_did_finder_lib/servicex_adaptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@


MAX_RETRIES = 3
CH_LEN = 30


class ServiceXAdapter:
Expand Down Expand Up @@ -92,10 +91,10 @@ def put_file_add(self, file_info):
self.logger.error(f'After {attempts} tries, failed to send ServiceX App a put_file '
f'message: {str(file_info)} - Ignoring error.')

def put_file_add_bulk(self, file_list):
def put_file_add_bulk(self, file_list, chunk_length=30):
# we first chunk up file_list as it can be very large in
# case there are a lot of replicas and a lot of files.
chunks = [file_list[i:i + CH_LEN] for i in range(0, len(file_list), CH_LEN)]
chunks = [file_list[i:i + chunk_length] for i in range(0, len(file_list), chunk_length)]
for chunk in chunks:
success = False
attempts = 0
Expand Down

0 comments on commit e4bf4bf

Please sign in to comment.