Final publish extended (#54)

* update CHANGELOG * Better publishing for all aligner indexes * fix kallisto * update snapshots * alternate publishing solution * properly fix merge conflicts * simplify source_gff * Update samtools.nf.test.snap
nf-core · Dec 6, 2024 · b692521 · b692521
1 parent 4db5898
commit b692521
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 78 deletions.
diff --git a/assets/schema_input.json b/assets/schema_input.json
@@ -21,8 +21,15 @@
             "source_vcf": {
                 "type": "string",
                 "errorMessage": "Where the references came from",
+                "default": "unknown",
                 "meta": ["source_vcf"]
             },
+            "source_gff": {
+                "type": "string",
+                "errorMessage": "Where the references came from",
+                "default": "unknown",
+                "meta": ["source_gff"]
+            },
             "species": {
                 "type": "string",
                 "errorMessage": "Species of the reference",

diff --git a/assets/test/default_extended.yml b/assets/test/default_extended.yml
@@ -5,6 +5,7 @@
   fasta_sizes: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/GRCh38_chr21.fa.sizes"
   gff: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/genes_chr21.gff"
   gtf: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/GRCh38_chr21.gtf"
+  source_gff: "CUSTOM"
   mito_name: "MT"
   readme: "https://raw.githubusercontent.com/nf-core/test-datasets/references/references/GRCh38_chr21/README.md"
   source: "nf-core/references"

diff --git a/main.nf b/main.nf
@@ -133,13 +133,25 @@ output {
         path { meta, intervals -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/Genes/${file}" } }
     }
     'hisat2_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/Hisat2Index/version2.2.1" } }
+        path { meta, index ->
+            { file ->
+                meta.source_gff == "unknown"
+                    ? "${meta.species}/${meta.source}/${meta.id}/Sequence/Hisat2Index/version2.2.1"
+                    : "${meta.species}/${meta.source}/${meta.id}/Sequence/Hisat2Index/${meta.source_gff}/version2.2.1"
+            }
+        }
     }
     'intervals_bed' {
         path { meta, intervals -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/intervals/${file}" } }
     }
     'kallisto_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/KallistoIndex/version0.51.1/${file}" } }
+        path { meta, index ->
+            { file ->
+                meta.source_gff == "unknown"
+                    ? "${meta.species}/${meta.source}/${meta.id}/Sequence/KallistoIndex/version0.51.1/${file}"
+                    : "${meta.species}/${meta.source}/${meta.id}/Sequence/KallistoIndex/${meta.source_gff}/version0.51.1/${file}"
+            }
+        }
     }
     'msisensorpro_list' {
         path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/msisensorpro/${file}" } }
@@ -154,16 +166,34 @@ output {
         path { folder -> { file -> "multiqc/multiqc_report" } }
     }
     'rsem_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/RSEMIndex/version1.3.1" } }
+        path { meta, index ->
+            { file ->
+                meta.source_gff == "unknown"
+                    ? "${meta.species}/${meta.source}/${meta.id}/Sequence/RSEMIndex/version1.3.1"
+                    : "${meta.species}/${meta.source}/${meta.id}/Sequence/RSEMIndex/${meta.source_gff}/version1.3.1"
+            }
+        }
     }
     'salmon_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/SalmonIndex/version1.10.3" } }
+        path { meta, index ->
+            { file ->
+                meta.source_gff == "unknown"
+                    ? "${meta.species}/${meta.source}/${meta.id}/Sequence/SalmonIndex/version1.10.3"
+                    : "${meta.species}/${meta.source}/${meta.id}/Sequence/SalmonIndex/${meta.source_gff}/version1.10.3"
+            }
+        }
     }
     'splice_sites' {
         path { meta, txt -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/SpliceSites/${file}" } }
     }
     'star_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/STARIndex/version2.7.11b" } }
+        path { meta, index ->
+            { file ->
+                meta.source_gff == "unknown"
+                    ? "${meta.species}/${meta.source}/${meta.id}/Sequence/STARIndex/version2.7.11b"
+                    : "${meta.species}/${meta.source}/${meta.id}/Sequence/STARIndex/${meta.source_gff}/version2.7.11b"
+            }
+        }
     }
     'transcript_fasta' {
         path { meta, fasta -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/TranscriptFasta/${file}" } }

diff --git a/modules.json b/modules.json
@@ -59,7 +59,8 @@
                     "hisat2/build": {
                         "branch": "master",
                         "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/hisat2/build/hisat2-build.diff"
                     },
                     "hisat2/extractsplicesites": {
                         "branch": "master",
@@ -69,7 +70,8 @@
                     "kallisto/index": {
                         "branch": "master",
                         "git_sha": "a1abf90966a2a4016d3c3e41e228bfcbd4811ccc",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/kallisto/index/kallisto-index.diff"
                     },
                     "msisensorpro/scan": {
                         "branch": "master",
@@ -107,7 +109,8 @@
                     "star/genomegenerate": {
                         "branch": "master",
                         "git_sha": "a5ad53288c79fa52c5ae708c317e09ec2dd149ab",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/star/genomegenerate/star-genomegenerate.diff"
                     },
                     "tabix/tabix": {
                         "branch": "master",

diff --git a/subworkflows/local/create_align_index_with_gff/main.nf b/subworkflows/local/create_align_index_with_gff/main.nf
@@ -57,6 +57,7 @@ workflow CREATE_ALIGN_INDEX_WITH_GFF {
 
             HISAT2_EXTRACTSPLICESITES(gtf_hisat2)
 
+            versions = versions.mix(HISAT2_EXTRACTSPLICESITES.out.versions)
             splice_sites = input_splice_sites.mix(HISAT2_EXTRACTSPLICESITES.out.txt)
 
             if (run_hisat2) {
@@ -68,7 +69,6 @@ workflow CREATE_ALIGN_INDEX_WITH_GFF {
 
                 hisat2_index = HISAT2_BUILD.out.index
 
-                versions = versions.mix(HISAT2_EXTRACTSPLICESITES.out.versions)
                 versions = versions.mix(HISAT2_BUILD.out.versions)
             }
         }

diff --git a/tests/.nftignore b/tests/.nftignore
@@ -1,11 +1,11 @@
 **/kallisto
-**/RSEMIndex/*/Log.out
-**/STARIndex/*/Log.out
-**/SalmonIndex/*/ctable.bin
-**/SalmonIndex/*/pos.bin
-**/SalmonIndex/*/pre_indexing.log
-**/SalmonIndex/*/ref_indexing.log
-**/SalmonIndex/*/seq.bin
+**/RSEMIndex/**/Log.out
+**/STARIndex/**/Log.out
+**/SalmonIndex/**/ctable.bin
+**/SalmonIndex/**/pos.bin
+**/SalmonIndex/**/pre_indexing.log
+**/SalmonIndex/**/ref_indexing.log
+**/SalmonIndex/**/seq.bin
 **/dragmap/*/hash_table.cfg
 **/dragmap/*/hash_table.cfg.bin
 **/dragmap/*/hash_table_stats.txt

diff --git a/tests/hisat2.nf.test.snap b/tests/hisat2.nf.test.snap
@@ -17,15 +17,16 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.1.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.2.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.3.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.4.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.5.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.6.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.7.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.8.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.1.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.2.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.3.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.4.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.5.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.6.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.7.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/CUSTOM/version2.2.1/GRCh38_chr21.8.ht2",
                 "multiqc",
                 "multiqc/multiqc_data",
                 "multiqc/multiqc_data/multiqc.log",
@@ -53,7 +54,7 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T15:35:30.36657085"
+        "timestamp": "2024-12-06T11:47:41.58238436"
     },
     "Run with profile test for hisat2 | --input GRCh38_chr21.yml": {
         "content": [
@@ -122,6 +123,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T15:36:59.914837757"
+        "timestamp": "2024-12-06T11:33:17.476760832"
     }
 }
diff --git a/tests/kallisto.nf.test.snap b/tests/kallisto.nf.test.snap
@@ -52,7 +52,7 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T15:55:35.924356749"
+        "timestamp": "2024-12-06T11:34:03.486585563"
     },
     "Run with profile test for kallisto | --input assets/test/default_extended.yml": {
         "content": [
@@ -72,8 +72,9 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1/kallisto",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/CUSTOM",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/CUSTOM/version0.51.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/CUSTOM/version0.51.1/kallisto",
                 "multiqc",
                 "multiqc/multiqc_data",
                 "multiqc/multiqc_data/multiqc.log",
@@ -93,6 +94,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T15:55:10.518283725"
+        "timestamp": "2024-12-06T11:50:22.597389596"
     }
 }
diff --git a/tests/rnaseq.nf.test.snap b/tests/rnaseq.nf.test.snap
@@ -119,6 +119,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T15:59:30.460893418"
+        "timestamp": "2024-12-06T11:37:48.115918724"
     }
 }
diff --git a/tests/rsem.nf.test.snap b/tests/rsem.nf.test.snap
@@ -18,31 +18,32 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/GRCh38_chr21.fa",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/Genome",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/Log.out",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/SA",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/SAindex",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/chrLength.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/chrName.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/chrNameLength.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/chrStart.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/exonGeTrInfo.tab",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/exonInfo.tab",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/geneInfo.tab",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.chrlist",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.grp",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.idx.fa",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.n2g.idx.fa",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.seq",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.ti",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genome.transcripts.fa",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/genomeParameters.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/sjdbInfo.txt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/sjdbList.fromGTF.out.tab",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/sjdbList.out.tab",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/version1.3.1/transcriptInfo.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/GRCh38_chr21.fa",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/Genome",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/Log.out",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/SA",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/SAindex",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/chrLength.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/chrName.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/chrNameLength.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/chrStart.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/exonGeTrInfo.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/exonInfo.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/geneInfo.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.chrlist",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.grp",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.idx.fa",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.n2g.idx.fa",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.seq",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.ti",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genome.transcripts.fa",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/genomeParameters.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/sjdbInfo.txt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/sjdbList.fromGTF.out.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/sjdbList.out.tab",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/RSEMIndex/CUSTOM/version1.3.1/transcriptInfo.tab",
                 "multiqc",
                 "multiqc/multiqc_data",
                 "multiqc/multiqc_data/multiqc.log",
@@ -85,7 +86,7 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T16:01:13.352587524"
+        "timestamp": "2024-12-06T11:54:36.556736231"
     },
     "Run with profile test for rsem | --input GRCh38_chr21.yml": {
         "content": [
@@ -187,6 +188,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-05T16:03:06.550798189"
+        "timestamp": "2024-12-06T11:41:35.396864169"
     }
 }