From 37e1fdadf1a3fc02e7c55853031743d560e26bc5 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Fri, 3 Jun 2022 15:07:35 -0500 Subject: [PATCH 1/6] just bulk no memcache --- requirements.txt | 2 +- src/servicex/did_finder/lookup_request.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0fe7ad5..c3d9060 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1a5 +servicex-did-finder-lib>=1.1a6 xmltodict wheel diff --git a/src/servicex/did_finder/lookup_request.py b/src/servicex/did_finder/lookup_request.py index cec3973..579537c 100644 --- a/src/servicex/did_finder/lookup_request.py +++ b/src/servicex/did_finder/lookup_request.py @@ -65,6 +65,7 @@ def lookup_files(self): avg_replicas = 0 lookup_start = datetime.now() + full_file_list = [] for ds_files in self.rucio_adapter.list_files_for_did(self.did): for af in ds_files: n_files += 1 @@ -72,7 +73,8 @@ def lookup_files(self): total_paths += len(af['paths']) if self.prefix: af['paths'] = [self.prefix+fp for fp in af['paths']] - yield af + full_file_list.append(af) + yield full_file_list lookup_finish = datetime.now() From 0d1a620a48b72df01a00131db7e41b5818cb165d Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Sat, 4 Jun 2022 12:33:09 -0500 Subject: [PATCH 2/6] fix lib version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c3d9060..44c64bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1a6 +servicex-did-finder-lib==1.1a6 xmltodict wheel From 10583e59207635ad6f4dff0138fdf27f5ec58118 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Sat, 4 Jun 2022 15:23:15 -0500 Subject: [PATCH 3/6] change in lib version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 44c64bc..bf158fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib==1.1a6 +servicex-did-finder-lib==1.1a7 xmltodict wheel From 0376d63306223435002d32f64417ccbc113e41ed Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Fri, 3 Jun 2022 15:07:35 -0500 Subject: [PATCH 4/6] Add files via batch interface instead of one at a time --- src/servicex/did_finder/lookup_request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/servicex/did_finder/lookup_request.py b/src/servicex/did_finder/lookup_request.py index cec3973..579537c 100644 --- a/src/servicex/did_finder/lookup_request.py +++ b/src/servicex/did_finder/lookup_request.py @@ -65,6 +65,7 @@ def lookup_files(self): avg_replicas = 0 lookup_start = datetime.now() + full_file_list = [] for ds_files in self.rucio_adapter.list_files_for_did(self.did): for af in ds_files: n_files += 1 @@ -72,7 +73,8 @@ def lookup_files(self): total_paths += len(af['paths']) if self.prefix: af['paths'] = [self.prefix+fp for fp in af['paths']] - yield af + full_file_list.append(af) + yield full_file_list lookup_finish = datetime.now() From 125163adf38d3e979df98b81d2aec21d8d2a56b5 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Sat, 4 Jun 2022 12:33:09 -0500 Subject: [PATCH 5/6] Pick up serviceX did finder lib that supports bulk --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f61fe1c..cfe9ef1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1.1 +servicex-did-finder-lib>=1.2 xmltodict wheel From 4fd597e29070773ae96480fd8024af70ddb8185b Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Wed, 8 Jun 2022 23:25:54 -0500 Subject: [PATCH 6/6] moved to lib version 1.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf158fc..01ad89a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib==1.1a7 +servicex-did-finder-lib==1.2 xmltodict wheel