From 8ca1b4fffeb9a0f86b66f2631e965424a6a5d968 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Tue, 5 Apr 2022 11:17:16 -0500 Subject: [PATCH 1/7] fixing CMS logical file issue --- src/servicex/did_finder/rucio_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/servicex/did_finder/rucio_adapter.py b/src/servicex/did_finder/rucio_adapter.py index 60ba873..a3a840f 100644 --- a/src/servicex/did_finder/rucio_adapter.py +++ b/src/servicex/did_finder/rucio_adapter.py @@ -143,7 +143,7 @@ def list_files_for_did(self, did): continue path = self.get_paths(f['url']) \ if not self.report_logical_files else \ - [f['identity']] + [f['identity'].strip('cms:')] g_files.append( { From b9a1845d1bd283d15b0b7a6b753066700a401848 Mon Sep 17 00:00:00 2001 From: Gordon Watts Date: Tue, 5 Apr 2022 16:10:51 -0400 Subject: [PATCH 2/7] Enable parsing of DID parameters (#51) * New version of DID library that parses `get` and `files` parameters * Reshape replica-not-found errors so that they are reported via an exception at the very end. See the [ServiceX story](https://github.com/ssl-hep/ServiceX/issues/395) for more background information. --- * Bump the library * Refix the version. Duh. * Fix a 3.6 bug in a library * Typeo in version number * Bump release - last one was bogus * Lastest bug fixes * Use the new get availibility code * Fix flake8 error * Bug fix in the client library * Move to released version of library --- requirements.txt | 2 +- src/servicex/did_finder/rucio_adapter.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0fe7ad5..9c8d85d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1a5 +servicex-did-finder-lib>=1.1 xmltodict wheel diff --git a/src/servicex/did_finder/rucio_adapter.py b/src/servicex/did_finder/rucio_adapter.py index 60ba873..eb8bd3c 100644 --- a/src/servicex/did_finder/rucio_adapter.py +++ b/src/servicex/did_finder/rucio_adapter.py @@ -120,6 +120,7 @@ def list_files_for_did(self, did): datasets = self.list_datasets_for_did(did) if not datasets: return + no_replica_files = 0 for ds in datasets: reps = self.replica_client.list_replicas( [{'scope': ds[0], 'name': ds[1]}], @@ -140,6 +141,7 @@ def list_files_for_did(self, did): # Path is either a list of replicas or a single logical name if 'url' not in f: self.logger.error(f"File {f['identity']} has no replicas.") + no_replica_files += 1 continue path = self.get_paths(f['url']) \ if not self.report_logical_files else \ @@ -154,3 +156,7 @@ def list_files_for_did(self, did): } ) yield g_files + + if no_replica_files > 0: + raise ValueError(f'Dataset {did} is missing replicas for {no_replica_files} ' + 'of its files.') From d3eed1c0aac3524b8e16c4d35b8ce3574ea9fd89 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Wed, 6 Apr 2022 12:55:41 -0500 Subject: [PATCH 3/7] just one more info log line --- src/servicex/did_finder/lookup_request.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/servicex/did_finder/lookup_request.py b/src/servicex/did_finder/lookup_request.py index cec3973..9814c24 100644 --- a/src/servicex/did_finder/lookup_request.py +++ b/src/servicex/did_finder/lookup_request.py @@ -64,6 +64,7 @@ def lookup_files(self): total_paths = 0 avg_replicas = 0 lookup_start = datetime.now() + self.logger.info("Starting Lookup of {}".format(self.did)) for ds_files in self.rucio_adapter.list_files_for_did(self.did): for af in ds_files: From 83dc79f4fd0a5a9c2aef57df15d64589b5d53246 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Wed, 6 Apr 2022 15:33:49 -0500 Subject: [PATCH 4/7] moving to lib 1.1.1 --- requirements.txt | 2 +- src/servicex/did_finder/lookup_request.py | 1 - src/servicex/did_finder/rucio_adapter.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0fe7ad5..f61fe1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1a5 +servicex-did-finder-lib>=1.1.1 xmltodict wheel diff --git a/src/servicex/did_finder/lookup_request.py b/src/servicex/did_finder/lookup_request.py index 9814c24..cec3973 100644 --- a/src/servicex/did_finder/lookup_request.py +++ b/src/servicex/did_finder/lookup_request.py @@ -64,7 +64,6 @@ def lookup_files(self): total_paths = 0 avg_replicas = 0 lookup_start = datetime.now() - self.logger.info("Starting Lookup of {}".format(self.did)) for ds_files in self.rucio_adapter.list_files_for_did(self.did): for af in ds_files: diff --git a/src/servicex/did_finder/rucio_adapter.py b/src/servicex/did_finder/rucio_adapter.py index a3a840f..60ba873 100644 --- a/src/servicex/did_finder/rucio_adapter.py +++ b/src/servicex/did_finder/rucio_adapter.py @@ -143,7 +143,7 @@ def list_files_for_did(self, did): continue path = self.get_paths(f['url']) \ if not self.report_logical_files else \ - [f['identity'].strip('cms:')] + [f['identity']] g_files.append( { From 09a08529e0a2f3b537c56a2d4ec72b02154b1713 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Thu, 7 Apr 2022 12:08:31 -0500 Subject: [PATCH 5/7] return of strip cms --- src/servicex/did_finder/rucio_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/servicex/did_finder/rucio_adapter.py b/src/servicex/did_finder/rucio_adapter.py index 60ba873..a3a840f 100644 --- a/src/servicex/did_finder/rucio_adapter.py +++ b/src/servicex/did_finder/rucio_adapter.py @@ -143,7 +143,7 @@ def list_files_for_did(self, did): continue path = self.get_paths(f['url']) \ if not self.report_logical_files else \ - [f['identity']] + [f['identity'].strip('cms:')] g_files.append( { From 0376d63306223435002d32f64417ccbc113e41ed Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Fri, 3 Jun 2022 15:07:35 -0500 Subject: [PATCH 6/7] Add files via batch interface instead of one at a time --- src/servicex/did_finder/lookup_request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/servicex/did_finder/lookup_request.py b/src/servicex/did_finder/lookup_request.py index cec3973..579537c 100644 --- a/src/servicex/did_finder/lookup_request.py +++ b/src/servicex/did_finder/lookup_request.py @@ -65,6 +65,7 @@ def lookup_files(self): avg_replicas = 0 lookup_start = datetime.now() + full_file_list = [] for ds_files in self.rucio_adapter.list_files_for_did(self.did): for af in ds_files: n_files += 1 @@ -72,7 +73,8 @@ def lookup_files(self): total_paths += len(af['paths']) if self.prefix: af['paths'] = [self.prefix+fp for fp in af['paths']] - yield af + full_file_list.append(af) + yield full_file_list lookup_finish = datetime.now() From 125163adf38d3e979df98b81d2aec21d8d2a56b5 Mon Sep 17 00:00:00 2001 From: Ilija Vukotic Date: Sat, 4 Jun 2022 12:33:09 -0500 Subject: [PATCH 7/7] Pick up serviceX did finder lib that supports bulk --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f61fe1c..cfe9ef1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ rucio-clients>=1.26.1 pika==1.1.0 -servicex-did-finder-lib>=1.1.1 +servicex-did-finder-lib>=1.2 xmltodict wheel