Merge branch 'ad7-caller-split' into ip13-caller-split

# Conflicts: # conf/hgi.config
wtsi-hgi · Mar 11, 2024 · b46969f · b46969f
2 parents 85f83f0 + 38a103d
commit b46969f
Show file tree

Hide file tree

Showing 27 changed files with 143 additions and 210 deletions.
diff --git a/conf/base.config b/conf/base.config
@@ -15,8 +15,9 @@ process {
     shell  = ['/bin/bash', '-euo', 'pipefail']
 
     // memory errors which should be retried. otherwise error out
-    errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
-    maxRetries    = 1
+    //errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
+    errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
+    maxRetries    = 3
     maxErrors     = '-1'
 
     // Process-specific resource requirements
@@ -71,12 +72,12 @@ process {
         memory          = { check_max( 30.GB * task.attempt, 'memory' ) }
     }
     withName: 'GATK4_MARKDUPLICATES|GATK4_MARKDUPLICATESSPARK' {
-        cpus           = { check_max( 6 * task.attempt, 'cpus' ) }
-        memory         = { check_max( 30.GB * task.attempt, 'memory' ) }
+        cpus           = { check_max( 1 * task.attempt, 'cpus' ) }
+        memory         = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
     withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_BASERECALIBRATOR_SPARK|GATK4_GATHERBQSRREPORTS'{
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
+        cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
     withName:'MOSDEPTH'{
         cpus            = { check_max( 4 * task.attempt, 'cpus' ) }

diff --git a/conf/hgi.config b/conf/hgi.config
@@ -8,13 +8,16 @@ process {
     }
 
     withName: 'GATK4_GENOMICSDBIMPORT' {
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 5.GB * task.attempt, 'memory' ) }
+        cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
+        clusterOptions  = { "-R \"rusage[tmp=20G]\"" }
+        ext.args        = "--batch-size 50 --reader-threads 1 -ip 500"
     }
 
     withName: 'GATK4_GENOTYPEGVCFS' {
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 12.GB * task.attempt, 'memory' ) }
+        cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
+        clusterOptions  = { "-R \"rusage[tmp=20G]\"" }
         ext.args        = { "-G StandardAnnotation -G AS_StandardAnnotation" }
     }
 
@@ -24,12 +27,12 @@ process {
     }
 
     withName:'GATK4_MERGEVCFS'{
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
+        cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
 
     withName: 'GATK4_HAPLOTYPECALLER' {
-    	cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
+    	cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
         memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
 

diff --git a/modules/local/create_intervals_bed/main.nf b/modules/local/create_intervals_bed/main.nf
@@ -28,11 +28,12 @@ process CREATE_INTERVALS_BED {
                 # no runtime estimate in this row, assume default value
                 t = (\$3 - \$2) / ${params.nucleotides_per_second}
             }
-            if (name == "" || (chunk > 600 && (chunk + t) > longest * 1.05)) {
+            if (name == "" || (chunk > 600 && (chunk + t) > longest * 1.00) || \$1 != chr) {
                 # start a new chunk
                 name = sprintf("%s_%d-%d.bed", \$1, \$2+1, \$3)
                 chunk = 0
                 longest = 0
+                chr = \$1
             }
             if (t > longest)
                 longest = t

diff --git a/modules/nf-core/gatk4/applybqsr/main.nf b/modules/nf-core/gatk4/applybqsr/main.nf
diff --git a/modules/nf-core/gatk4/applyvqsr/main.nf b/modules/nf-core/gatk4/applyvqsr/main.nf
diff --git a/modules/nf-core/gatk4/baserecalibrator/main.nf b/modules/nf-core/gatk4/baserecalibrator/main.nf
diff --git a/modules/nf-core/gatk4/calculatecontamination/main.nf b/modules/nf-core/gatk4/calculatecontamination/main.nf
diff --git a/modules/nf-core/gatk4/cnnscorevariants/main.nf b/modules/nf-core/gatk4/cnnscorevariants/main.nf
diff --git a/modules/nf-core/gatk4/createsequencedictionary/main.nf b/modules/nf-core/gatk4/createsequencedictionary/main.nf
diff --git a/modules/nf-core/gatk4/estimatelibrarycomplexity/main.nf b/modules/nf-core/gatk4/estimatelibrarycomplexity/main.nf
diff --git a/modules/nf-core/gatk4/filtermutectcalls/main.nf b/modules/nf-core/gatk4/filtermutectcalls/main.nf
diff --git a/modules/nf-core/gatk4/filtervarianttranches/main.nf b/modules/nf-core/gatk4/filtervarianttranches/main.nf
diff --git a/modules/nf-core/gatk4/gatherbqsrreports/main.nf b/modules/nf-core/gatk4/gatherbqsrreports/main.nf
diff --git a/modules/nf-core/gatk4/gatherpileupsummaries/main.nf b/modules/nf-core/gatk4/gatherpileupsummaries/main.nf
diff --git a/modules/nf-core/gatk4/genomicsdbimport/main.nf b/modules/nf-core/gatk4/genomicsdbimport/main.nf
diff --git a/modules/nf-core/gatk4/genotypegvcfs/main.nf b/modules/nf-core/gatk4/genotypegvcfs/main.nf
diff --git a/modules/nf-core/gatk4/getpileupsummaries/main.nf b/modules/nf-core/gatk4/getpileupsummaries/main.nf
diff --git a/modules/nf-core/gatk4/haplotypecaller/main.nf b/modules/nf-core/gatk4/haplotypecaller/main.nf
diff --git a/modules/nf-core/gatk4/intervallisttobed/main.nf b/modules/nf-core/gatk4/intervallisttobed/main.nf
diff --git a/modules/nf-core/gatk4/learnreadorientationmodel/main.nf b/modules/nf-core/gatk4/learnreadorientationmodel/main.nf
diff --git a/modules/nf-core/gatk4/markduplicates/main.nf b/modules/nf-core/gatk4/markduplicates/main.nf
diff --git a/modules/nf-core/gatk4/mergemutectstats/main.nf b/modules/nf-core/gatk4/mergemutectstats/main.nf
diff --git a/modules/nf-core/gatk4/mergevcfs/main.nf b/modules/nf-core/gatk4/mergevcfs/main.nf
diff --git a/modules/nf-core/gatk4/mutect2/main.nf b/modules/nf-core/gatk4/mutect2/main.nf
diff --git a/modules/nf-core/gatk4/variantrecalibrator/main.nf b/modules/nf-core/gatk4/variantrecalibrator/main.nf
diff --git a/nextflow.config b/nextflow.config
@@ -242,8 +242,6 @@ profiles {
         executor.memory        = 8.GB
         executor.name          = 'local'
     }
-    // Custom profile
-    hgi                        { includeConfig 'conf/hgi.config' }
     // Basic test profile for CI
     test                       { includeConfig 'conf/test.config' }
     test_cache                 { includeConfig 'conf/test/cache.config' }
@@ -389,6 +387,9 @@ includeConfig 'conf/modules/post_variant_calling.config'
 //annotate
 includeConfig 'conf/modules/annotate.config'
 
+// Custom profile
+includeConfig 'conf/hgi.config'
+
 // Function to ensure that resource requirements don't go beyond
 // a maximum limit
 def check_max(obj, type) {