Merge branch 'bulk_file_additions' of https://github.com/ssl-hep/Serv…

…iceX_DID_Finder_Rucio into bulk_file_additions
ssl-hep · Jun 9, 2022 · 5e84a1e · 5e84a1e
2 parents 4fd597e + 125163a
commit 5e84a1e
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 rucio-clients>=1.26.1
 pika==1.1.0
-servicex-did-finder-lib==1.2
+servicex-did-finder-lib>=1.2
 xmltodict
 wheel
diff --git a/src/servicex/did_finder/rucio_adapter.py b/src/servicex/did_finder/rucio_adapter.py
@@ -120,6 +120,7 @@ def list_files_for_did(self, did):
         datasets = self.list_datasets_for_did(did)
         if not datasets:
             return
+        no_replica_files = 0
         for ds in datasets:
             reps = self.replica_client.list_replicas(
                 [{'scope': ds[0], 'name': ds[1]}],
@@ -140,10 +141,11 @@ def list_files_for_did(self, did):
                     # Path is either a list of replicas or a single logical name
                     if 'url' not in f:
                         self.logger.error(f"File {f['identity']} has no replicas.")
+                        no_replica_files += 1
                         continue
                     path = self.get_paths(f['url']) \
                         if not self.report_logical_files else \
-                        [f['identity']]
+                        [f['identity'].strip('cms:')]
 
                     g_files.append(
                         {
@@ -154,3 +156,7 @@ def list_files_for_did(self, did):
                         }
                     )
             yield g_files
+
+        if no_replica_files > 0:
+            raise ValueError(f'Dataset {did} is missing replicas for {no_replica_files} '
+                             'of its files.')