wtsi-hgi · y-popov · Oct 19, 2022 · Oct 25, 2022 · Oct 25, 2022 · Oct 25, 2022
diff --git a/conf/base.config b/conf/base.config
@@ -9,41 +9,42 @@
 */
 
 process {
+    cache  = 'lenient'
     cpus   = { check_max( 1    * task.attempt, 'cpus'   ) }
-    memory = { check_max( 6.GB * task.attempt, 'memory' ) }
-    time   = { check_max( 4.h  * task.attempt, 'time'   ) }
+    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+    //time   = { check_max( 12.h  * task.attempt, 'time'   ) }
     shell  = ['/bin/bash', '-euo', 'pipefail']
 
     // memory errors which should be retried. otherwise error out
-    errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' }
-    maxRetries    = 1
+    errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
+    maxRetries    = 4
     maxErrors     = '-1'
 
     // Process-specific resource requirements
     // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
     withLabel:process_single {
         cpus          = { check_max( 1                  , 'cpus'    ) }
-        memory        = { check_max( 6.GB * task.attempt, 'memory'  ) }
-        time          = { check_max( 4.h  * task.attempt, 'time'    ) }
+        memory        = { check_max( 8.GB * task.attempt, 'memory'  ) }
+        //time          = { check_max( 12.h  * task.attempt, 'time'    ) }
     }
     withLabel:process_low {
         cpus          = { check_max( 2     * task.attempt, 'cpus'    ) }
         memory        = { check_max( 12.GB * task.attempt, 'memory'  ) }
-        time          = { check_max( 4.h   * task.attempt, 'time'    ) }
+        //time          = { check_max( 12.h   * task.attempt, 'time'    ) }
     }
     withLabel:process_medium {
         cpus          = { check_max( 6     * task.attempt, 'cpus'    ) }
         memory        = { check_max( 36.GB * task.attempt, 'memory'  ) }
-        time          = { check_max( 8.h   * task.attempt, 'time'    ) }
+        //time          = { check_max( 12.h   * task.attempt, 'time'    ) }
     }
     withLabel:process_high {
         cpus          = { check_max( 12    * task.attempt, 'cpus'    ) }
         memory        = { check_max( 72.GB * task.attempt, 'memory'  ) }
-        time          = { check_max( 16.h  * task.attempt, 'time'    ) }
-    }
-    withLabel:process_long {
-        time          = { check_max( 20.h  * task.attempt, 'time'    ) }
+        //time          = { check_max( 48.h  * task.attempt, 'time'    ) }
     }
+    //withLabel:process_long {
+        //time          = { check_max( 48.h  * task.attempt, 'time'    ) }
+    //}
     withLabel:process_high_memory {
         memory        = { check_max( 200.GB * task.attempt, 'memory' ) }
     }
@@ -71,12 +72,18 @@ process {
         memory          = { check_max( 30.GB * task.attempt, 'memory' ) }
     }
     withName: 'GATK4_MARKDUPLICATES|GATK4_MARKDUPLICATESSPARK' {
-        cpus           = { check_max( 6 * task.attempt, 'cpus' ) }
-        memory         = { check_max( 30.GB * task.attempt, 'memory' ) }
+        cpus           = { check_max( 1                 , 'cpus' ) }
+        memory         = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
-    withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_BASERECALIBRATOR_SPARK|GATK4_GATHERBQSRREPORTS'{
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
+    withName: 'GATK4_GENOMICSDBIMPORT|GATK4_GENOTYPEGVCFS' {
+        cpus           = { check_max( 1                 , 'cpus' ) }
+        memory         = { check_max( 12.GB * task.attempt, 'memory' ) }
+        queue          = 'basement'
+    }
+
+    withName:'GATK4_HAPLOTYPECALLER|GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_BASERECALIBRATOR_SPARK|GATK4_GATHERBQSRREPORTS'{
+        cpus            = { check_max( 1                , 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
     withName:'MOSDEPTH'{
         cpus            = { check_max( 4 * task.attempt, 'cpus' ) }
@@ -94,11 +101,25 @@ process {
         memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
     }
     withName:'GATK4_MERGEVCFS'{
-        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
-        memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
+        cpus            = { check_max( 1               , 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
     }
     withName: 'MULTIQC' {
         cpus            = { check_max( 4 * task.attempt, 'cpus' ) }
         memory          = { check_max( 12.GB * task.attempt, 'memory' ) }
     }
 }
+
+executor {
+    name = 'lsf'
+    queueSize = 4000
+    poolSize = 4
+    submitRateLimit = '10 sec'
+    killBatchSize = 50
+    pollInterval = '10 sec'
+    queueStatInterval = '20 sec'
+    dumpInterval = '10 sec'
+    exitReadTimeout= '10 sec'
+    perJobMemLimit=true
+}
+
diff --git a/conf/lsf.conf b/conf/lsf.conf
@@ -0,0 +1,30 @@
+process {
+    cache = 'lenient'
+    executor = 'lsf'
+    shell = ['/bin/bash', '-euo', 'pipefail']
+}
+
+executor {
+    name = 'lsf'
+    queueSize = 4000
+    poolSize = 4
+    submitRateLimit = '10 sec'
+    killBatchSize = 50
+    pollInterval = '10 sec'
+    queueStatInterval = '20 sec'
+    dumpInterval = '10 sec'
+    exitReadTimeout= '10 sec'
+    perJobMemLimit=true
+}
+
+docker {
+    enabled = false
+}
+
+singularity {
+    enabled     = true
+    autoMounts  = true
+    cacheDir = '/lustre/scratch118/humgen/resources/containers/'
+    runOptions = '--dns 172.18.255.1,172.18.255.2,172.18.255.3'
+    envWhitelist = 'HOSTNAME,SSH_CONNECTION,SSH_CLIENT,CVS_RSH,http_proxy,https_proxy,HTTP_PROXY,HTTPS_PROXY'
+}
diff --git a/conf/modules.config b/conf/modules.config
@@ -727,11 +727,11 @@ process{
     }
 
     withName: 'HAPLOTYPECALLER' {
-        ext.args         = { params.joint_germline ? "-ERC GVCF" : "" }
+        ext.args         = { params.joint_germline ? "-ERC GVCF -G StandardAnnotation -G AS_StandardAnnotation -G StandardHCAnnotation" : "" }
         ext.prefix       = { meta.num_intervals <= 1 ? ( params.joint_germline ? "${meta.id}.haplotypecaller.g" : "${meta.id}.haplotypecaller" ) : ( params.joint_germline ? "${meta.id}.haplotypecaller.${intervals.simpleName}.g" :"${meta.id}.haplotypecaller.${intervals.simpleName}" ) }
         ext.when         = { params.tools && params.tools.split(',').contains('haplotypecaller') }
         publishDir       = [
-            enabled: !params.joint_germline,
+            //enabled: !params.joint_germline,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/"},
             pattern: "*{vcf.gz,vcf.gz.tbi}",
@@ -762,6 +762,7 @@ process{
     }
 
     withName: 'GATK4_GENOTYPEGVCFS' {
+        ext.args         = { "-G StandardAnnotation -G AS_StandardAnnotation" }
         ext.prefix       = { meta.num_intervals > 1 ? meta.intervals_name : "joint_germline" }
     }
 
@@ -784,17 +785,19 @@ process{
 
     withName: 'VARIANTRECALIBRATOR_INDEL' {
         ext.prefix = { "${meta.id}_INDEL" }
-        ext.args = "-an QD -an MQRankSum -an ReadPosRankSum -an FS -an SOR -an DP  -mode INDEL"
+        ext.args = "-AS -an QD -an MQRankSum -an ReadPosRankSum -an FS -an SOR -an DP  -mode INDEL"
         publishDir = [
             enabled: false
         ]
     }
 
     withName: 'VARIANTRECALIBRATOR_SNP' {
         ext.prefix = { "${meta.id}_SNP" }
-        ext.args = "-an QD -an MQ -an MQRankSum -an ReadPosRankSum -an FS -an SOR -mode SNP"
+        ext.args = "-AS -an QD -an MQ -an MQRankSum -an ReadPosRankSum -an FS -an SOR -mode SNP"
         publishDir = [
-            enabled: false
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/variant_calling/haplotypecaller/joint_variant_calling/"},
+            pattern: "*{R.pdf,R}"
         ]
     }
 
@@ -804,12 +807,8 @@ process{
     }
 
     withName: 'GATK4_APPLYVQSR_INDEL'{
-        ext.prefix    = { "${meta.id}_INDEL" }
+        ext.prefix    = { "joint_germline_recalibrated" }
         ext.args      = '--truth-sensitivity-filter-level 99.9 -mode INDEL'
-    }
-
-    withName: 'MERGE_VQSR' {
-        ext.prefix       = "joint_germline_recalibrated"
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/haplotypecaller/joint_variant_calling/"},
@@ -818,6 +817,16 @@ process{
         ]
     }
 
+//    withName: 'MERGE_VQSR' {
+//        ext.prefix       = "joint_germline_recalibrated"
+//        publishDir       = [
+//            mode: params.publish_dir_mode,
+//            path: { "${params.outdir}/variant_calling/haplotypecaller/joint_variant_calling/"},
+//            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+//            pattern: "*{vcf.gz,vcf.gz.tbi}"
+//        ]
+//    }
+
     // MANTA
     withName: 'MERGE_MANTA.*' {
         publishDir       = [

diff --git a/conf/test_full_germline.config b/conf/test_full_germline.config
@@ -15,10 +15,15 @@ params {
     config_profile_description = 'Full test dataset to check germline VC pipeline function'
 
     // Input data for full size test
-    input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/NA12878_WGS_30x_full_test.csv'
+    input = '/lustre/scratch123/hgi/teams/hgi/nextflow_ci_staging/novaseq_eval/novaseq.csv'
 
     // Other params
     tools = 'strelka,freebayes,haplotypecaller,deepvariant,manta,tiddit,cnvkit,vep'
+    fasta = '/lustre/scratch125/humgen/resources/ref/Homo_sapiens/HS38DH/hs38DH.fa'
+    fasta_fai = '/lustre/scratch125/humgen/resources/ref/Homo_sapiens/HS38DH/hs38DH.fa.fai'
+    dict = '/lustre/scratch125/humgen/resources/ref/Homo_sapiens/HS38DH/hs38DH.dict'
+    intervals = '/lustre/scratch125/humgen/resources/ref/Homo_sapiens/HS38DH/wgs_hg38.even.handcurated.20k.FINAL.bed'
+    nucleotides_per_second = 40000
 
     split_fastq = 50000000
 }
diff --git a/modules/local/create_intervals_bed/main.nf b/modules/local/create_intervals_bed/main.nf
@@ -28,11 +28,12 @@ process CREATE_INTERVALS_BED {
                 # no runtime estimate in this row, assume default value
                 t = (\$3 - \$2) / ${params.nucleotides_per_second}
             }
-            if (name == "" || (chunk > 600 && (chunk + t) > longest * 1.05)) {
+            if (name == "" || (chunk > 600 && (chunk + t) > longest * 1.00) || \$1 != chr ) {
                 # start a new chunk
                 name = sprintf("%s_%d-%d.bed", \$1, \$2+1, \$3)
                 chunk = 0
                 longest = 0
+                chr = \$1
             }
             if (t > longest)
                 longest = t

diff --git a/modules/nf-core/modules/deepvariant/main.nf b/modules/nf-core/modules/deepvariant/main.nf
diff --git a/modules/nf-core/modules/gatk4/applybqsr/main.nf b/modules/nf-core/modules/gatk4/applybqsr/main.nf
diff --git a/modules/nf-core/modules/gatk4/applybqsrspark/main.nf b/modules/nf-core/modules/gatk4/applybqsrspark/main.nf
diff --git a/modules/nf-core/modules/gatk4/applyvqsr/main.nf b/modules/nf-core/modules/gatk4/applyvqsr/main.nf
diff --git a/modules/nf-core/modules/gatk4/baserecalibrator/main.nf b/modules/nf-core/modules/gatk4/baserecalibrator/main.nf
diff --git a/modules/nf-core/modules/gatk4/baserecalibratorspark/main.nf b/modules/nf-core/modules/gatk4/baserecalibratorspark/main.nf
diff --git a/modules/nf-core/modules/gatk4/calculatecontamination/main.nf b/modules/nf-core/modules/gatk4/calculatecontamination/main.nf
diff --git a/modules/nf-core/modules/gatk4/cnnscorevariants/main.nf b/modules/nf-core/modules/gatk4/cnnscorevariants/main.nf
diff --git a/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf b/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf
diff --git a/modules/nf-core/modules/gatk4/estimatelibrarycomplexity/main.nf b/modules/nf-core/modules/gatk4/estimatelibrarycomplexity/main.nf