Merge pull request #362 from broadinstitute/jb-h5ad-de

Extending differential expression analysis to AnnData files (SCP-5764)
broadinstitute · Aug 26, 2024 · 858bb96 · 858bb96
2 parents b820bbb + 3ea1d42
commit 858bb96
Show file tree

Hide file tree

Showing 7 changed files with 19,096 additions and 6 deletions.
diff --git a/ingest/cli_parser.py b/ingest/cli_parser.py
@@ -9,7 +9,7 @@
 
 
 # Ingest file types
-EXPRESSION_FILE_TYPES = ["dense", "mtx", "loom"]
+EXPRESSION_FILE_TYPES = ["dense", "mtx", "h5ad"]
 
 
 def bq_dataset_exists(dataset):

diff --git a/ingest/de.py b/ingest/de.py
@@ -197,8 +197,8 @@ def execute_de(self):
                     self.genes_path,
                     self.barcodes_path,
                 )
-            elif self.matrix_file_type == "dense":
-                DifferentialExpression.de_logger.info("preparing DE on dense matrix")
+            elif self.matrix_file_type == "dense" or self.matrix_file_type == "h5ad":
+                DifferentialExpression.de_logger.info(f"preparing DE on {self.matrix_file_type} matrix")
                 self.run_scanpy_de(
                     self.cluster,
                     self.metadata,
@@ -211,7 +211,7 @@ def execute_de(self):
                     self.method,
                 )
             else:
-                msg = f"Submitted matrix_file_type should be \"dense\" or \"mtx\" not \"{self.matrix_file_type}\""
+                msg = f"Submitted matrix_file_type should be \"dense\", \"mtx\" or \"h5ad\" not \"{self.matrix_file_type}\""
                 print(msg)
                 log_exception(
                     DifferentialExpression.dev_logger,
@@ -337,14 +337,21 @@ def run_scanpy_de(
             matrix_object = IngestFiles(matrix_file_path, None)
             local_file_path = matrix_object.resolve_path(matrix_file_path)[1]
             adata = sc.read(local_file_path)
+        elif matrix_file_type == "h5ad":
+            matrix_object = IngestFiles(matrix_file_path, None)
+            local_file_path = matrix_object.resolve_path(matrix_file_path)[1]
+            adata = matrix_object.open_anndata(local_file_path)
         else:
             # MTX reconstitution UNTESTED (SCP-4203)
             # will want try/except here to catch failed data object composition
             adata = DifferentialExpression.adata_from_mtx(
                 matrix_file_path, genes_path, barcodes_path
             )
 
-        adata = adata.transpose()
+        if not matrix_file_type == "h5ad":
+            adata = adata.transpose()
+
+        use_raw = True if matrix_file_type == "h5ad" else False
 
         adata = DifferentialExpression.subset_adata(adata, de_cells)
 
@@ -362,7 +369,7 @@ def run_scanpy_de(
                 adata,
                 annotation,
                 key_added=rank_key,
-                use_raw=False,
+                use_raw=use_raw,
                 method=method,
                 pts=True,
             )

diff --git a/ingest/ingest_pipeline.py b/ingest/ingest_pipeline.py
@@ -65,6 +65,10 @@
 
 # Differential expression analysis (sparse matrix)
 python ingest_pipeline.py --study-id addedfeed000000000000000 --study-file-id dec0dedfeed1111111111111 differential_expression --annotation-name cell_type__ontology_label --annotation-type group --annotation-scope study --matrix-file-path ../tests/data/differential_expression/sparse/sparsemini_matrix.mtx --gene-file ../tests/data/differential_expression/sparse/sparsemini_features.tsv --barcode-file ../tests/data/differential_expression/sparse/sparsemini_barcodes.tsv --matrix-file-type mtx --annotation-file ../tests/data/differential_expression/sparse/sparsemini_metadata.txt --cluster-file ../tests/data/differential_expression/sparse/sparsemini_cluster.txt --cluster-name de_sparse_integration --study-accession SCPsparsemini --differential-expression
+
+# Differential expression analysis (h5ad matrix)
+python ingest_pipeline.py --study-id addedfeed000000000000000 --study-file-id dec0dedfeed1111111111111 differential_expression --annotation-name cell_type__ontology_label --annotation-type group --annotation-scope study --matrix-file-path ../tests/data/differential_expression/de_dense_matrix.tsv --matrix-file-type h5ad --annotation-file ../tests/data/differential_expression/de_dense_metadata.tsv --cluster-file ../tests/data/differential_expression/de_dense_cluster.tsv --cluster-name de_integration --study-accession SCPdev --differential-expression
+
 """
 import json
 import logging