From 6d4599ae8decbe15f447a62a4e194cc13a8b162a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 24 Apr 2018 19:55:04 +0200 Subject: [PATCH 1/6] Started basic work on Travis Tests --- .travis.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.travis.yml b/.travis.yml index e69de29..3df68a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -0,0 +1,30 @@ +sudo: required +language: java +jdk: openjdk8 +services: + - docker +python: + - "2.7" +cache: pip +matrix: + fast_finish: true + +install: + # Install Nextflow + - mkdir /tmp/nextflow + - cd /tmp/nextflow + - wget -qO- get.nextflow.io | bash + - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + # Install nf-core/tools + - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools + - cd /tmp/nf-core-tools + - pip install --user -e . + # Reset + - cd ${TRAVIS_BUILD_DIR}/tests + +env: + - ALIGNER=bwa NXF_VER=0.27.6 + - ALIGNER=bwa + +script: + - "nf-core lint ${TRAVIS_BUILD_DIR}" \ No newline at end of file From 40b1a5412f12551d077555066f3c875060ba336e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 24 Apr 2018 19:58:41 +0200 Subject: [PATCH 2/6] Added Build Status Badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ddf553..6e1403c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # ![nf-core/ExoSeq](https://raw.githubusercontent.com/nf-core/Exoseq/master/docs/images/ExoSeq_logo.png) - +[![Build Status](https://travis-ci.org/nf-core/ExoSeq.svg?branch=master)](https://travis-ci.org/nf-core/ExoSeq) [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) [![Gitter](https://img.shields.io/badge/gitter-%20join%20chat%20%E2%86%92-4fb99a.svg)](https://gitter.im/nf-core/Lobby) [![Docker Container available](https://img.shields.io/docker/automated/nfcore/exoseq.svg)](https://hub.docker.com/r/nfcore/exoseq/) From 6f37838aa384f508dbdcb3a83286445d591d9c0e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 24 Apr 2018 22:21:21 +0200 Subject: [PATCH 3/6] Work in progress --- .travis.yml | 23 +++++++++++++++-------- scripts/install.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 scripts/install.sh diff --git a/.travis.yml b/.travis.yml index 3df68a9..8532fdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,26 +5,33 @@ services: - docker python: - "2.7" + +node_js: "node" + cache: pip + matrix: fast_finish: true install: - # Install Nextflow - - mkdir /tmp/nextflow - - cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + - "./scripts/install.sh --tool $TOOL_INSTALL" # Install nf-core/tools - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools - cd /tmp/nf-core-tools - pip install --user -e . + # Get Kit Files + - wget -O kits.tar.bz2 https://qbic-intranet.am10.uni-tuebingen.de/owncloud/index.php/s/Qvku3etEqb3PW58/download + # Get tiny Reference Files # Reset - cd ${TRAVIS_BUILD_DIR}/tests env: - - ALIGNER=bwa NXF_VER=0.27.6 - - ALIGNER=bwa + global: + - NXF_VER=0.27.6 SGT_VER=2.4.2 + matrix: + - PROFILE=docker TEST=TOOLS TOOL_INSTALL=ALL + - PROFILE=singularity TEST=TOOLS TOOL_INSTALL=ALL script: - - "nf-core lint ${TRAVIS_BUILD_DIR}" \ No newline at end of file + - "nf-core lint ${TRAVIS_BUILD_DIR}" + - "./scripts/test.sh --profile $PROFILE --test $TEST" \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 0000000..205814d --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#Credit to the Sarek devs at https://github.com/SciLifeLab/Sarek +TOOL="all" + +while [[ $# -gt 0 ]] +do + key="$1" + case $key in + -t|--tool) + TOOL="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + shift # past argument + ;; + esac +done + +# Install Nextflow +if [[ "$TOOL" = nextflow ]] || [[ "$TOOL" = all ]] +then + cd $HOME + curl -fsSL get.nextflow.io | bash + chmod +x nextflow + sudo mv nextflow /usr/local/bin/ +fi + +# Install Singularity +if [[ "$TOOL" = singularity ]] || [[ "$TOOL" = all ]] +then + sudo apt-get install squashfs-tools + cd $HOME + wget https://github.com/singularityware/singularity/releases/download/$SGT_VER/singularity-$SGT_VER.tar.gz + tar xvf singularity-$SGT_VER.tar.gz + cd singularity-$SGT_VER + ./configure --prefix=/usr/local + make + sudo make install + cd .. + rm -rf singularity-$SGT_VER* +fi \ No newline at end of file From dbb1cbc44ee9e3e877d436bbc473a72ac5825be5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 25 Apr 2018 09:56:52 +0200 Subject: [PATCH 4/6] Removed node from .travis.yml --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8532fdf..a3b83b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,6 @@ services: python: - "2.7" -node_js: "node" - cache: pip matrix: @@ -34,4 +32,4 @@ env: script: - "nf-core lint ${TRAVIS_BUILD_DIR}" - - "./scripts/test.sh --profile $PROFILE --test $TEST" \ No newline at end of file + - "./scripts/test.sh --profile $PROFILE --test $TEST" From 95f705bffaf8a4096fa512e9369a19a88ce09da1 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 7 Aug 2018 16:07:26 +0200 Subject: [PATCH 5/6] Minimal changelog changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b26c391..a643b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,2 +1,2 @@ -## [nf-core/ExoSeq v0.9dev] +## [nf-core/ExoSeq v1.0dev] The release marks the point where the pipeline was moved from SciLifeLab/NGI-ExoSeq over to the new nf-core community, at nf-core/ExoSeq. From 123eb33347e4faf8fc4dc2f87b0a76bb5573d262 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 7 Aug 2018 16:24:53 +0200 Subject: [PATCH 6/6] Add tracedir --- PairedSingleSampleWF.nf | 48 ++++++++--------------------------------- conf/metafiles.config | 18 ++++++++++++++++ conf/test.config | 23 ++++++++++++++++++++ environment.yml | 8 +++---- nextflow.config | 28 ++++++++++++++++++------ 5 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 conf/metafiles.config create mode 100644 conf/test.config diff --git a/PairedSingleSampleWF.nf b/PairedSingleSampleWF.nf index 383bb8d..d08b32b 100644 --- a/PairedSingleSampleWF.nf +++ b/PairedSingleSampleWF.nf @@ -43,22 +43,9 @@ Output: --outdir Path where the results to be saved [Default: './results'] Kit files: +--kitfiles Path to kitfiles defined in metafiles.config +--metafiles Path to metafiles defined in metafiles.config --kit Kit used to prep samples [Default: 'agilent_v5'] ---bait Absolute path to bait file ---target Absolute path to target file ---target_bed Absolute path to target bed file (snpEff compatible format) - -Genome/Variation files: ---dbsnp Absolute path to dbsnp file ---thousandg Absolute path to 1000G file ---mills Absolute path to Mills file ---omni Absolute path to Omni file ---gfasta Absolute path to genome fasta file ---bwa_index Absolute path to bwa genome index - -Other options: ---exome Exome data, if this is not set, run as genome data ---project Uppnex project to user for SLURM executor For more detailed information regarding the parameters and usage refer to package documentation at https://github.com/nf-core/ExoSeq @@ -69,9 +56,7 @@ params.name = false params.help = false params.reads = false params.singleEnd = false -params.genome = false params.run_id = false -params.exome = false //default genome, set to true to run restricting to exome positions params.aligner = 'bwa' //Default, but stay tuned for later ;-) params.saveReference = true @@ -91,24 +76,14 @@ params.three_prime_clip_r2 = 0 // Kit options params.kit = 'agilent_v5' -params.bait = params.kitFiles[ params.kit ] ? params.kitFiles[ params.kit ].bait ?: false : false -params.target = params.kitFiles[ params.kit ] ? params.kitFiles[ params.kit ].target ?: false : false -params.target_bed = params.kitFiles[ params.kit ] ? params.kitFiles[ params.kit ].target_bed ?: false : false -params.dbsnp = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].dbsnp ?: false : false -params.thousandg = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].thousandg ?: false : false -params.mills = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].mills ?: false : false -params.omni = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].omni ?: false : false -params.gfasta = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].gfasta ?: false : false -params.bwa_index = params.metaFiles[ params.genome ] ? params.metaFiles[ params.genome ].bwa_index ?: false : false // Has the run name been specified by the user? -// this has the bonus effect of catching both -name and --name +// this has the bonus effect of catching both -name and --name custom_runName = params.name if( !(workflow.runName ==~ /[a-z]+_[a-z]+/) ){ custom_runName = workflow.runName } - // Show help when needed if (params.help){ log.info helpMessage @@ -119,14 +94,12 @@ if (params.help){ if (!params.reads || !params.genome){ exit 1, "Parameters '--reads' and '--genome' are required to run the pipeline" } -if (!params.kitFiles[ params.kit ] && ['bait', 'target'].count{ params[it] } != 2){ - exit 1, "Kit '${params.kit}' is not available in pre-defined config, so " + - "provide all kit specific files with option '--bait' and '--target'" +if (!params.kitfiles){ + exit 1, "No Exome Capturing Kit specified!" +} +if (!params.metafiles){ + exit 1, "No Exome Metafiles specified!" } - if (!params.metaFiles[ params.genome ] && ['gfasta', 'bwa_index', 'dbsnp', 'thousandg', 'mills', 'omni'].count{ params[it] } != 6){ - exit 1, "Genome '${params.genome}' is not available in pre-defined config, so you need to provide all genome specific " + - "files with options '--gfasta', '--bwa_index', '--dbsnp', '--thousandg', '--mills' and '--omni'" - } // Create a channel for input files @@ -137,7 +110,6 @@ Channel // Validate Input indices for BWA Mem and GATK - if(params.aligner == 'bwa' ){ bwaId = Channel .fromPath("${params.gfasta}.bwt") @@ -150,8 +122,7 @@ def summary = [:] summary['Run Name'] = custom_runName ?: workflow.runName summary['Reads'] = params.reads summary['Data Type'] = params.singleEnd ? 'Single-End' : 'Paired-End' -summary['Genome'] = params.genome -summary['WES/WGS'] = params.exome ? 'WES' : 'WGS' +summary['Genome Assembly'] = params.genome summary['Trim R1'] = params.clip_r1 summary['Trim R2'] = params.clip_r2 summary["Trim 3' R1"] = params.three_prime_clip_r1 @@ -178,7 +149,6 @@ summary['Config Profile'] = workflow.profile log.info summary.collect { k,v -> "${k.padRight(15)}: $v" }.join("\n") log.info "=========================================" - try { if( ! workflow.nextflow.version.matches(">= $params.nf_required_version") ){ throw GroovyException('Nextflow version too old') diff --git a/conf/metafiles.config b/conf/metafiles.config new file mode 100644 index 0000000..c7ae48f --- /dev/null +++ b/conf/metafiles.config @@ -0,0 +1,18 @@ + +genomes { + 'GRCh37' { + gfasta = '${params.metafiles}/b37/human_g1k_v37.fasta' + bwa_index = '${params.metafiles}/b37/human_g1k_v37.fasta.bwt' + dbsnp = '${params.metafiles}/b37/dbsnp_138.b37.vcf' + mills = '${params.metafiles}/b37/Mills_and_1000G_gold_standard.indels.b37.vcf' + omni = '${params.metafiles}/b37/1000G_omni2.5.b37.vcf' + thousandg = '${params.metafiles}/b37/1000G_phase1.snps.high_confidence.b37.vcf' + kits { + 'agilent_v5' { + bait = '${params.kitfiles}/agilent_v5/S04380110_Regions.interval_list' + target = '${params.kitfiles}/agilent_v5/S04380110_Covered.interval_list' + target_bed = '${params.kitfiles}/agilent_v5/S04380110_Covered.bed' + } + } + } + } \ No newline at end of file diff --git a/conf/test.config b/conf/test.config new file mode 100644 index 0000000..f11799d --- /dev/null +++ b/conf/test.config @@ -0,0 +1,23 @@ +/* + * ------------------------------------------------- + * Nextflow config file for running tests + * ------------------------------------------------- + * Defines bundled input files and everything required + * to run a fast and simple test. Use as follows: + * nextflow run nf-core/rnaseq -profile test + */ + +params { + max_cpus = 2 + max_memory = 6.GB + max_time = 48.h + // Input data + singleEnd = false + kitfiles = './kits' + readPaths = [ + ['Testdata_R1', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz']], + ['Testdata_R2', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']], + ] + // Genome references + fasta = 'https://github.com/nf-core/test-datasets/raw/exoseq/reference/human_g1k_v37_decoy.small.fasta' +} diff --git a/environment.yml b/environment.yml index d7a4df3..5a429a3 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ # You can use this file to create a conda environment for this pipeline: # conda env create -f environment.yml -name: nfcore-exoseq-0.9dev +name: nfcore-exoseq-1.0dev channels: - bioconda - conda-forge @@ -11,7 +11,7 @@ dependencies: - bwa=0.7.17 - fastqc=0.11.7 - trim-galore=0.4.5 - - samtools=1.8 - - gatk4=4.0.3.0 + - samtools=1.9 + - gatk4=4.0.4.0 - qualimap=2.2.2a - - multiqc=1.5 + - multiqc=1.6 diff --git a/nextflow.config b/nextflow.config index 8fff529..ef09244 100644 --- a/nextflow.config +++ b/nextflow.config @@ -11,9 +11,12 @@ params { outdir = './results' - version = '0.9dev' - nf_required_version = '0.27.6' + version = '1.0dev' + nf_required_version = '0.30.2' container = 'nfcore/exoseq' + metafiles = './references' + kitfiles = './kitfiles' + tracedir = "${params.outdir}/pipeline_info" } @@ -35,21 +38,34 @@ profiles { uppmax { includeConfig 'conf/uppmax.config' } + test { + includeConfig 'conf/base.config' + includeConfig 'conf/test.config' + includeConfig 'conf/metafiles.config' + } } // Capture exit codes from upstream processes when piping -process.shell = ['/bin/bash','-euo', 'pipefail'] +process.shell = ['/bin/bash', '-euo', 'pipefail'] timeline { enabled = true - file = "${params.outdir}/NGI-ExoSeq_timeline.html" + file = "${params.tracedir}/pipeline_info/nfcore-exoseq_timeline.html" +} +report { + enabled = true + file = "${params.tracedir}/pipeline_info/nfcore-exoseq_report.html" } - trace { enabled = true - file = "${params.outdir}/NGI-ExoSeq_trace.txt" + file = "${params.tracedir}/pipeline_info/nfcore-exoseq_trace.txt" +} +dag { + enabled = true + file = "${params.tracedir}/pipeline_info/nfcore-exoseq_DAG.svg" } + manifest { homePage = 'https://github.com/SciLifeLab/NGI-ExoSeq' description = 'Nextflow Exome Sequencing Best Practice analysis pipeline.'